SUI-1820 - Write Confident matches back to eclipse

src/SUI.Client/SUI.Client.GraphQLProcessJob/Infrastructure/GraphQLProcessor.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+
 using Eclipse.GraphQL;
 
 using Microsoft.Extensions.Logging;
@@ -7,6 +9,7 @@
 
 using SUI.Client.Core.Application.Interfaces;
 using SUI.Client.Core.Application.Models;
+using SUI.Client.Core.Application.UseCases.MatchPeople;
 using SUI.Client.Core.Infrastructure.CsvParsers;
 
 namespace SUI.Client.GraphQLProcessJob.Infrastructure;
@@ -20,84 +23,177 @@
 {
     public async Task RunAsync(CancellationToken cancellationToken)
     {
+        var timer = Stopwatch.StartNew();
         logger.LogInformation("Running Graph QL Process Job.");
+        var mappings = csvMatchDataOptions.Value.ColumnMappings;
+
+        List<CsvRecordDto> csvRecords = await FetchAndCompilePersonRecordsAsync(mappings, cancellationToken);
+
+        logger.LogInformation("Completed compiling GraphQL records. Total records retrieved: {Count}. Elapsed Time: {ElapsedTime}",
+            csvRecords.Count, timer.Elapsed.ToString("g"));
+
+        var matchedResults = await matchPersonRecordOrchestrator.ProcessAsync(
+            csvRecords,
+            "graphql_extract",
+            cancellationToken
+        );
+
+        logger.LogInformation(
+            "Finished processing matching. Result count: {Count}. Matches: {MatchCount}. Elapsed time: {ElapsedTime}",
+            matchedResults.Count, matchedResults.Count(x => x.ApiResult is
+            {
+                Result.IsHighConfidenceMatch: true
+            }), timer.Elapsed.ToString("g"));
+
+        await SaveMatchedNhsNumbersAsync(matchedResults, mappings, cancellationToken);
+
+        logger.LogInformation("GraphQL Job Complete. Total time: {ElapsedTime}", timer.Elapsed.ToString("g"));
+    }
+
+    private async Task<List<CsvRecordDto>> FetchAndCompilePersonRecordsAsync(
+        CsvMatchDataOptions.Headers mappings,
+        CancellationToken cancellationToken)
+    {
         int pageNumber = 1;
-        const int pageSize = 10;
-        bool hasMoreResults = true;
+        const int pageSize = 100;
         var csvRecords = new List<CsvRecordDto>();
-        var mappings = csvMatchDataOptions.Value.ColumnMappings;
 
-        while (hasMoreResults && !cancellationToken.IsCancellationRequested)
+        while (!cancellationToken.IsCancellationRequested)
         {
             var results = await eclipseClient.PersonByCriteria.ExecuteAsync(options.Value.MaxAge,
                 new RequestCursorInput { PageNumber = pageNumber, PageSize = pageSize }, cancellationToken);
             results.EnsureNoErrors();
 
-            if (results.Data?.PersonByCriteria?.Results is { Count: > 0 } resultsList)
+            var resultsList = results.Data?.PersonByCriteria?.Results;
+            if (resultsList == null || resultsList.Count == 0)
             {
-                foreach (var result in resultsList)
-                {
-                    if (result is not IPersonByCriteria_PersonByCriteria_Results_Person person)
-                    {
-                        continue;
-                    }
-
-                    var personDictionary = new Dictionary<string, string>
-                    {
-                        { mappings.Id, person.Id },
-                        { mappings.Given, person.Forename ?? "" },
-                        { mappings.Family, person.Surname ?? "" },
-                        { mappings.BirthDate, person.DateOfBirth?.Lower?.ToString(csvMatchDataOptions.Value.DateFormat) ?? "" },
-                        {
-                            mappings.Postcode,
-                            person.Addresses.FirstOrDefault(a => a.Id == person.PreferredAddress?.Id)?.Location
-                                ?.Postcode ?? ""
-                        }
-                    };
-
-                    if (!string.IsNullOrEmpty(mappings.NhsNumber))
-                    {
-                        personDictionary[mappings.NhsNumber] = person.NhsNumber ?? "";
-                    }
-
-                    if (!string.IsNullOrEmpty(mappings.Gender))
-                    {
-                        personDictionary[mappings.Gender] = person.Gender?.ToString().ToLower() ?? "";
-                    }
-
-                    csvRecords.Add(new CsvRecordDto(personDictionary));
-                }
+                break;
+            }
 
-                var cursor = results.Data?.PersonByCriteria?.Cursor;
-                if (cursor != null && cursor.Offset + cursor.Returned < cursor.TotalSize)
-                {
-                    pageNumber++;
-                }
-                else
+            foreach (var result in resultsList)
+            {
+                if (result is IPersonByCriteria_PersonByCriteria_Results_Person person)
                 {
-                    hasMoreResults = false;
+                    csvRecords.Add(new CsvRecordDto(MapPersonToDictionary(person, mappings)));
                 }
             }
-            else
+
+            var cursor = results.Data?.PersonByCriteria?.Cursor;
+            if (cursor != null && logger.IsEnabled(LogLevel.Information))
             {
-                hasMoreResults = false;
+                logger.LogInformation(
+                    "Processed Page: {Page}. Page size: {PageSize}. Total Records: {TotalRecords}",
+                    cursor.PageNumber, cursor.PageSize, cursor.TotalSize);
             }
+
+            if (cursor == null || cursor.Offset + cursor.Returned >= cursor.TotalSize)
+            {
+                break;
+            }
+
+            pageNumber++;
         }
 
-        logger.LogInformation("Completed compiling GraphQL records. Total records retrieved: {Count}.",
-            csvRecords.Count);
+        return csvRecords;
+    }
 
-        var matchedResults = await matchPersonRecordOrchestrator.ProcessAsync(
-            csvRecords,
-            "graphql_extract",
-            cancellationToken
-        );
+    private Dictionary<string, string> MapPersonToDictionary(
+        IPersonByCriteria_PersonByCriteria_Results_Person person,
+        CsvMatchDataOptions.Headers mappings)
+    {
+        var personDictionary = new Dictionary<string, string>
+        {
+            { mappings.Id, person.Id },
+            { mappings.Given, person.Forename ?? "" },
+            { mappings.Family, person.Surname ?? "" },
+            { mappings.BirthDate, person.DateOfBirth?.Lower?.ToString(csvMatchDataOptions.Value.DateFormat) ?? "" },
+            { mappings.Postcode, GetPreferredPostcode(person) },
+            { "__ObjectVersion", person.ObjectVersion.ToString() },
+            { "__PersonTypes", string.Join(",", person.PersonTypes ?? []) }
+        };
 
-        logger.LogInformation(
-            "Finished processing matching with orchestrator. Result count: {Count}. Matches: {MatchCount}",
-            matchedResults.Count, matchedResults.Count(x => x.ApiResult is
+        if (!string.IsNullOrEmpty(mappings.NhsNumber))
+        {
+            personDictionary[mappings.NhsNumber] = person.NhsNumber ?? "";
+        }
+
+        if (!string.IsNullOrEmpty(mappings.Gender))
+        {
+            personDictionary[mappings.Gender] = person.Gender?.ToString().ToLower() ?? "";
+        }
+
+        return personDictionary;
+    }
+
+    private static string GetPreferredPostcode(IPersonByCriteria_PersonByCriteria_Results_Person person) =>
+        person.Addresses.FirstOrDefault(a => a.Id == person.PreferredAddress?.Id)?.Location?.Postcode ?? "";
+
+    private async Task SaveMatchedNhsNumbersAsync(
+        IEnumerable<ProcessedMatchRecord<CsvRecordDto>> matchedResults,
+        CsvMatchDataOptions.Headers mappings,
+        CancellationToken cancellationToken)
+    {
+        foreach (var result in matchedResults)
+        {
+            if (result.ApiResult is not { Result.IsHighConfidenceMatch: true } ||
+                string.IsNullOrEmpty(result.ApiResult.Result.NhsNumber))
             {
-                Result.IsHighConfidenceMatch: true
-            }));
+                continue;
+            }
+
+            var personId = result.OriginalData.Record[mappings.Id];
+            var matchedNhsNumber = result.ApiResult.Result.NhsNumber;
+
+            var existingNhsNumber = !string.IsNullOrEmpty(mappings.NhsNumber) &&
+                                    result.OriginalData.Record.TryGetValue(mappings.NhsNumber, out var extNhs)
+                                    ? extNhs
+                                    : null;
+
+            if (!string.IsNullOrEmpty(existingNhsNumber))
+            {
+                logger.LogInformation("Person {PersonId} already has NHS number. Skipping update.",
+                    personId);
+                continue;
+            }
+
+            if (!result.OriginalData.Record.TryGetValue("__ObjectVersion", out var objVerStr) ||
+                !int.TryParse(objVerStr, out var objectVersion))
+            {
+                logger.LogWarning("Could not find ObjectVersion for Person {PersonId}. Skipping NHS number update.", personId);
+                continue;
+            }
+
+            logger.LogInformation("Saving matched NHS number {NhsNumber} for Person {PersonId} with ObjectVersion {ObjectVersion}.",
+                matchedNhsNumber, personId, objectVersion);
+
+            try
+            {
+                var personTypesList = new List<PersonType>();
+                if (result.OriginalData.Record.TryGetValue("__PersonTypes", out var typesStr) &&
+                    !string.IsNullOrEmpty(typesStr))
+                {
+                    personTypesList = typesStr.Split(',', StringSplitOptions.RemoveEmptyEntries)
+                        .Select(Enum.Parse<PersonType>)
+                        .ToList();
+                }
+
+                var updateInput = new UpdatePerson
+                {
+                    Id = personId,
+                    NhsNumber = matchedNhsNumber,
+                    ObjectVersion = objectVersion,
+                    PersonTypes = personTypesList
+                };
+
+                var updateResult = await eclipseClient.UpdatePerson.ExecuteAsync(updateInput, cancellationToken);
+                updateResult.EnsureNoErrors();
+
+                logger.LogInformation("Successfully saved NHS number for Person {PersonId}.", personId);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to save NHS number for Person {PersonId}.", personId);
+            }
+        }
     }
 }

src/SUI.Client/SUI.Client.GraphQLProcessJob/PersonByCriteria.graphql

@@ -22,6 +22,8 @@ query PersonByCriteria($maxAge: Int, $paging: RequestCursorInput) {
                     mask
                 }
                 id
+                objectVersion
+                personTypes
                 forename
                 surname
                 gender

src/SUI.Client/SUI.Client.GraphQLProcessJob/UpdatePerson.graphql

@@ -0,0 +1,7 @@
+mutation UpdatePerson($input: UpdatePerson!) {
+    updatePerson(input: $input) {
+        id
+        objectVersion
+        nhsNumber
+    }
+}

src/SUI.Client/SUI.Client.GraphQLProcessJob/appsettings.json

@@ -2,7 +2,8 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.Hosting.Lifetime": "Information"
+      "Microsoft.Hosting.Lifetime": "Information",
+      "System.Net.Http.HttpClient": "Error"
     }
   },
   "GraphQlProcessJob": {