Salesforce Field Values in Sitecore Default Facets

using System;
using FuseIT.Sitecore.SalesforceConnector.Entities;
using FuseIT.Sitecore.SalesforceConnector.SalesforcePartner;
using FuseIT.Sitecore.SalesforceConnector;
using FuseIT.Sitecore.Salesforce;
using FuseIT.Sitecore.SalesforceConnector.DataSource;
using FuseIT.Sitecore.SalesforceConnector.Soql;
using MoreLinq;
using Sitecore.Configuration;
using Sitecore.Diagnostics;
using Sitecore.XConnect;
using Sitecore.XConnect.Client;
using Sitecore.XConnect.Client.Configuration;
using Sitecore.XConnect.Collection.Model;
using System.Collections.Generic;
using System.Linq;
using S4SContact = FuseIT.Sitecore.SalesforceConnector.Entities.Contact;
using S4SEntityBase = FuseIT.Sitecore.SalesforceConnector.Entities.EntityBase;
using S4SLead = FuseIT.Sitecore.SalesforceConnector.Entities.Contact;

namespace FuseIT.S4S.WebToSalesforce
{
    public class BulkUpdateXdbFromSalesforce
    {
        /// <summary>
        /// Get leads and contacts in Salesforce with a Sitecore visitor id that have been changed recently and updates the xDB
        /// </summary>
        public static void BulkUpdateRecentlyChanged()
        {
            Log.Info("Bulk updating xDB with recently changed Salesforce contacts and leads", typeof(BulkUpdateXdbFromSalesforce));

            var salesforceSession = new SalesforceSession("S4SConnString");
            var leadDataSource = new LeadDataSource(salesforceSession);
            leadDataSource.AddDataSourceFilter(visitorIdField, ComparisonOperator.NotEquals, null);
            leadDataSource.AddDataSourceFilter("isConverted", ComparisonOperator.Equals, false);
            leadDataSource.AddDataSourceFilter("LastModifiedDate", ComparisonOperator.GreaterOrEqual, DateTime.Now.AddDays(-2));

            var contactDataSource = new ContactDataSource(salesforceSession);
            contactDataSource.AddDataSourceFilter(visitorIdField, ComparisonOperator.NotEquals, null);
            contactDataSource.AddDataSourceFilter("LastModifiedDate", ComparisonOperator.GreaterOrEqual, DateTime.Now.AddDays(-2));

            BulkUpdate(leadDataSource, contactDataSource);
        }

        /// <summary>
        /// Takes every lead and contact in Salesforce with a Sitecore visitor id and updates the xDB
        /// </summary>
        public static void BulkUpdateAll()
        {
            Log.Info("Bulk updating xDB with all Salesforce contacts and leads", typeof(BulkUpdateXdbFromSalesforce));

            var salesforceSession = new SalesforceSession("S4SConnString");

            var leadDataSource = new LeadDataSource(salesforceSession);
            leadDataSource.AddDataSourceFilter(visitorIdField, ComparisonOperator.NotEquals, null);
            leadDataSource.AddDataSourceFilter("isConverted", ComparisonOperator.Equals, false);

            var contactDataSource = new ContactDataSource(salesforceSession);
            contactDataSource.AddDataSourceFilter(visitorIdField, ComparisonOperator.NotEquals, null);

           BulkUpdate(leadDataSource, contactDataSource);
        }

        static void BulkUpdate(LeadDataSource leadDataSource, ContactDataSource contactDataSource)
        {
            // "Id" is required by S4S when requesting existing Salesforce objects but unused by this code
            var fields = new[] { "Id", visitorIdField, "FirstName", "LastName", "Email" };

            var leads = LazyGetEntities<S4SLead>(leadDataSource, fields)
            .Select(lead => lead.InternalFields);

            var contacts = LazyGetEntities<S4SContact>(contactDataSource, fields)
            .Select(contact => contact.InternalFields);

            var leadsOrContacts = contacts.Concat(leads)
            .Select(internalFields => new LeadContactUnion
            {
                VisitorId = Guid.Parse(internalFields[visitorIdField]),
                FirstName = internalFields["FirstName"],
                LastName = internalFields["LastName"],
                Email = internalFields["Email"]
            });

            var contactExpandOptions = new ContactExpandOptions(PersonalInformation.DefaultFacetKey, EmailAddressList.DefaultFacetKey);
            using (var xconnect = SitecoreXConnectClientConfiguration.GetClient())
            {
                foreach (var batch in leadsOrContacts.Batch(100).ToList())
                {
                    // Get all xDB contacts associated with the visitor id stored in Salesforce (in this batch). 
                    // These are the xDB contacts we want to ensure are up to date
                    var contactReferences = ContactReferenceList(batch, aliasIdentifierSource, leadOrContact => leadOrContact.VisitorId.ToString());
                    var xcontactsDictionary = GetMultipleContacts(xconnect, contactReferences, contactExpandOptions)
                    .ToDictionary(contact => Guid.Parse(Identifier(contact, aliasIdentifierSource)));

                    // Get all contacts in this batch associated with a list manager for later merging
                    // When importing contacts in Sitecore's list manager, they're created with a ListManager identifier, and not automatically
                    // associated with existing contacts sharing the same email. The editor can choose another identifierSource and identifier, so Sitecore
                    // can merge things correctly
                    var listManagerXcontactReferences = ContactReferenceList(batch, listManagerIdentifierSource, leadOrContact => leadOrContact.Email);
                    var listManagerXcontacts = GetMultipleContacts(xconnect, listManagerXcontactReferences, contactExpandOptions)
                    .ToLookup(contact => Identifier(contact, listManagerIdentifierSource));

                    // Get all S4S contacts matching the emails in this batch. This is so we can check if we're about to update the S4S
                    // identifier to a value that is already assigned to another contact. Identifiers must be unique.
                    // Salesforce allows for leads/contacts with same email addresss, but since Sitecore identifiers must be unique, 
                    // two xDB contacts cannot share the same identifier.
                    var s4sXcontactReferences = ContactReferenceList(batch, s4sIdentifierSource, leadOrContact => leadOrContact.Email);
                    var s4sXcontacts = GetMultipleContacts(xconnect, s4sXcontactReferences, contactExpandOptions)
                    .ToLookup(contact => Identifier(contact, s4sIdentifierSource));

                    foreach (var leadOrContact in batch)
                    {
                        if (xcontactsDictionary.TryGetValue(leadOrContact.VisitorId, out Contact xcontact))
                        {
                            // Update personal info
                            var personalInfo = xcontact.GetFacet<PersonalInformation>(PersonalInformation.DefaultFacetKey) ?? new PersonalInformation();
                            personalInfo.FirstName = leadOrContact.FirstName;
                            personalInfo.LastName = leadOrContact.LastName;
                            xconnect.SetFacet(xcontact, PersonalInformation.DefaultFacetKey, personalInfo);

                            // Update email
                            var emailAddresses = xcontact.GetFacet<EmailAddressList>(EmailAddressList.DefaultFacetKey);
                            if (emailAddresses != null)
                            {
                               emailAddresses.PreferredEmail = new EmailAddress(leadOrContact.Email, true);
                               emailAddresses.PreferredKey = "Work";
                            }
                            else
                            {
                               emailAddresses = new EmailAddressList(new EmailAddress(leadOrContact.Email, true), "Work");
                            }
                            xconnect.SetFacet(xcontact, EmailAddressList.DefaultFacetKey, emailAddresses);

                            // Merge imported contacts (in the list manager) with this contact
                            var otherListManagerXcontacts = listManagerXcontacts[leadOrContact.Email]
                            .Where(listManagerXcontact => listManagerXcontact.Id != xcontact.Id)
                            .ToList();
                            foreach (var listManagerXcontact in otherListManagerXcontacts)
                            {
                               var isS4SContact = listManagerXcontact.Identifiers.Any(identifier => identifier.Source == s4sIdentifierSource);
                               if (!isS4SContact)
                               xconnect.MergeContacts(listManagerXcontact, xcontact);
                            }

                            // Check if email has changed and if so, update S4S identifier.
                            var otherS4SXcontacts = s4sXcontacts[leadOrContact.Email]
                            .Where(s4sXcontact => s4sXcontact.Id != xcontact.Id)
                            .ToList();

                            // Even though the existing S4S modifier might be different from what is stored, we cannot update it if another
                            // xDB contact uses the same identifier. Duplicate identifiers are not allowed.
                            if (!otherS4SXcontacts.Any())
                            {
                                var existingS4SIdentifier = NullableIdentifier(xcontact, s4sIdentifierSource);
                                if (existingS4SIdentifier != leadOrContact.Email)
                                {
                                    if (!string.IsNullOrEmpty(existingS4SIdentifier))
                                        xconnect.RemoveContactIdentifier(xcontact, s4sIdentifierSource, existingS4SIdentifier);

                                    xconnect.AddContactIdentifier(xcontact,
                                    new ContactIdentifier(s4sIdentifierSource, leadOrContact.Email, ContactIdentifierType.Known));
                                }
                           }

                           // Check if email has changed and if so, update ListManager identifier...
                           var existingListManagerIdentifier = NullableIdentifier(xcontact, listManagerIdentifierSource);
                           if (existingListManagerIdentifier != leadOrContact.Email)
                           {
                               if (!string.IsNullOrEmpty(existingListManagerIdentifier))
                                    xconnect.RemoveContactIdentifier(xcontact, listManagerIdentifierSource, existingListManagerIdentifier);

                               // ... but only if we're not merging with other xDB contacts, in which case their ListManager identifier will be transfered
                               if (!otherListManagerXcontacts.Any())
                                   xconnect.AddContactIdentifier(xcontact,
                                       new ContactIdentifier(listManagerIdentifierSource, leadOrContact.Email, ContactIdentifierType.Known));
                           }
                        }
                    }
                    xconnect.Submit();
                }
            }

            Log.Info("Bulk update succeeded", typeof(BulkUpdateXdbFromSalesforce));
        }

        // Sitecore agent needs to be able to initialize an object and call an instance member
        public void SitecoreAgentBulkUpdateRecentlyChanged() => BulkUpdateAll();

        class LeadContactUnion
        {
            public string FirstName;
            public string LastName;
            public string Email;
            public Guid VisitorId;
        }

        #region Constants

        static string s4sIdentifierSource = Settings.GetSetting("S4S.Analytics.IdentifierSource", "S4S");
        const string visitorIdField = "FuseITAnalytics__SitecoreVisitorId__c";
        // The alias identifier is what's stored in Salesforce by S4S as Sitecore Visitor Id
        const string aliasIdentifierSource = Constants.AliasIdentifierSource;
        const string listManagerIdentifierSource = Sitecore.ListManagement.Constants.ListManagerIdentifierSource;

        #endregion

        #region Helper methods

        /// <summary>
        /// Throws if identifier doesn't exist
        /// </summary>
        static string Identifier(Contact contact, string identifierSource) =>
           contact.Identifiers.First(identifier => identifier.Source == identifierSource).Identifier;

        static string NullableIdentifier(Contact contact, string identifierSource) =>
           contact.Identifiers.FirstOrDefault(identifier => identifier.Source == identifierSource)?.Identifier;

        static IReadOnlyCollection<IdentifiedContactReference> ContactReferenceList<T>(
            IEnumerable<T> list,
            string identifierSource,
            Func<T, string> identifier) =>
            list.Select(t => new IdentifiedContactReference(identifierSource, identifier(t))).ToList();

        static IEnumerable<Contact> GetMultipleContacts(
           XConnectClient xconnect,
            IReadOnlyCollection<IEntityReference<Contact>> contactReferences,
            ContactExpandOptions expandOptions) =>
                XConnectSynchronousExtensions.SuspendContextLock(() =>
                xconnect.GetContactsAsync(contactReferences, expandOptions))
                .Where(lookupResult => lookupResult.Exists)
                .Select(lookupResult => lookupResult.Entity);

        static IEnumerable<T> LazyGetEntities<T>(SalesforceDataSource dataSource, params string[] fields)
            where T : S4SEntityBase
        {
            var pager = dataSource.GetPager(fields);
            for (var index = 0; index < pager.TotalRecordCount; index++)
            yield return dataSource.EntityFromQueryResultPager<T>(pager, index);
        }

        #endregion
    }
}