diff --git a/src/Dfe.Analytics.EFCore/AnalyticsDeployer.cs b/src/Dfe.Analytics.EFCore/AnalyticsDeployer.cs index f20f4d0..f4c1fae 100644 --- a/src/Dfe.Analytics.EFCore/AnalyticsDeployer.cs +++ b/src/Dfe.Analytics.EFCore/AnalyticsDeployer.cs @@ -21,6 +21,7 @@ public async Task DeployAsync( string airbyteConnectionId, string? airbyteSyncMode, string hiddenPolicyTagName, + IReadOnlyDictionary additionalPolicyTags, IProgressReporter? progressReporter = null, CancellationToken cancellationToken = default) { @@ -46,7 +47,7 @@ await WithProgressReportingAsync( "Waiting for sync to complete"); await WithProgressReportingAsync( - () => UpdateBigQueryPolicyTagsAsync(configuration, hiddenPolicyTagName, progressReporter, cancellationToken), + () => UpdateBigQueryPolicyTagsAsync(configuration, hiddenPolicyTagName, additionalPolicyTags, progressReporter, cancellationToken), progressReporter, "Updating BigQuery policy tags"); } @@ -207,6 +208,7 @@ async Task WaitForCompletionAsync() internal async Task UpdateBigQueryPolicyTagsAsync( DatabaseSyncConfiguration configuration, string hiddenPolicyTagName, + IReadOnlyDictionary additionalPolicyTags, IProgressReporter progressReporter, CancellationToken cancellationToken = default) { @@ -269,7 +271,13 @@ await WithProgressReportingAsync( if (column.Hidden) { - bqField.PolicyTags.Names.Add(hiddenPolicyTagName); + var policyTagName = column.PolicyTag is string policyTag && policyTag != "hidden" + ? additionalPolicyTags.TryGetValue(column.PolicyTag, out var tag) + ? tag + : throw new InvalidOperationException($"Missing policy tag mapping for '{policyTag}'.") + : hiddenPolicyTagName; + + bqField.PolicyTags.Names.Add(policyTagName); } var policyTagNamesChanged = !existingPolicyTagNames.SetEquals(bqField.PolicyTags.Names); diff --git a/src/Dfe.Analytics.EFCore/Cli/Commands.Config.Apply.cs b/src/Dfe.Analytics.EFCore/Cli/Commands.Config.Apply.cs index bb876b2..b8cadd5 100644 --- a/src/Dfe.Analytics.EFCore/Cli/Commands.Config.Apply.cs +++ b/src/Dfe.Analytics.EFCore/Cli/Commands.Config.Apply.cs @@ -1,3 +1,4 @@ +using System.Collections.Frozen; using System.CommandLine; using Dfe.Analytics.EFCore.Configuration; using Microsoft.EntityFrameworkCore; @@ -19,6 +20,35 @@ public static Command GetConfigApplyCommand() var projectIdOption = new Option("--project-id") { Required = true }; var datasetIdOption = new Option("--dataset-id") { Required = true }; var hiddenPolicyTagNameOption = new Option("--hidden-policy-tag-name") { Required = true }; + var policyTagOption = new Option>("--policy-tag") + { + Required = false, + Arity = new ArgumentArity(2, ArgumentArity.OneOrMore.MaximumNumberOfValues), + AllowMultipleArgumentsPerToken = true, + CustomParser = result => + { + var policyTagMapping = new Dictionary(); + + if (result.Tokens.Count % 2 != 0) + { + result.AddError("--policy-tag must be specified as an alias followed by a name"); + return policyTagMapping; + } + + for (var i = 0; i < result.Tokens.Count; i += 2) + { + var alias = result.Tokens[i].Value; + var name = result.Tokens[i + 1].Value; + + if (!policyTagMapping.TryAdd(alias, name)) + { + result.AddError($"--policy-tag with alias '{alias}' has already been specified"); + } + } + + return policyTagMapping; + } + }; // Airbyte options var airbyteApiBaseAddressOption = new Option("--airbyte-api-base-address") { Required = true }; @@ -40,6 +70,7 @@ public static Command GetConfigApplyCommand() projectIdOption, datasetIdOption, hiddenPolicyTagNameOption, + policyTagOption, airbyteApiBaseAddressOption, airbyteClientIdOption, airbyteClientSecretOption, @@ -59,6 +90,7 @@ public static Command GetConfigApplyCommand() var projectId = parseResult.GetRequiredValue(projectIdOption); var datasetId = parseResult.GetRequiredValue(datasetIdOption); var hiddenPolicyTagName = parseResult.GetRequiredValue(hiddenPolicyTagNameOption); + var additionalPolicyTags = parseResult.GetValue(policyTagOption) ?? FrozenDictionary.Empty; var airbyteApiBaseAddress = parseResult.GetRequiredValue(airbyteApiBaseAddressOption); var airbyteClientId = parseResult.GetRequiredValue(airbyteClientIdOption); var airbyteClientSecret = parseResult.GetRequiredValue(airbyteClientSecretOption); @@ -100,6 +132,7 @@ await analyticsDeployer.DeployAsync( airbyteConnectionId, airbyteSyncMode, hiddenPolicyTagName, + additionalPolicyTags, cancellationToken: cts.Token); }); diff --git a/src/Dfe.Analytics.EFCore/Configuration/AnalyticsConfigurationProvider.cs b/src/Dfe.Analytics.EFCore/Configuration/AnalyticsConfigurationProvider.cs index c828e3f..a427e20 100644 --- a/src/Dfe.Analytics.EFCore/Configuration/AnalyticsConfigurationProvider.cs +++ b/src/Dfe.Analytics.EFCore/Configuration/AnalyticsConfigurationProvider.cs @@ -118,7 +118,10 @@ private static IReadOnlyCollection GetColumns(IEntityType entity columnSyncInfos.Add(new ColumnSyncInfo { Name = columnName, - Hidden = hidden + Hidden = hidden, + PolicyTag = hidden ? + columnSyncMetadata?.PolicyTag ?? tableSyncMetadata.DefaultColumnSettings.PolicyTag : + null }); } } diff --git a/src/Dfe.Analytics.EFCore/Configuration/ColumnSyncInfo.cs b/src/Dfe.Analytics.EFCore/Configuration/ColumnSyncInfo.cs index c55d25e..9516ff6 100644 --- a/src/Dfe.Analytics.EFCore/Configuration/ColumnSyncInfo.cs +++ b/src/Dfe.Analytics.EFCore/Configuration/ColumnSyncInfo.cs @@ -4,6 +4,7 @@ public sealed class ColumnSyncInfo : IEquatable { public required string Name { get; init; } public required bool Hidden { get; init; } + public required string? PolicyTag { get; init; } public override bool Equals(object? obj) { @@ -22,7 +23,7 @@ public override bool Equals(object? obj) public override int GetHashCode() { - return HashCode.Combine(Name, Hidden); + return HashCode.Combine(Name, Hidden, PolicyTag); } public bool Equals(ColumnSyncInfo? other) @@ -37,6 +38,6 @@ public bool Equals(ColumnSyncInfo? other) return true; } - return Name == other.Name && Hidden == other.Hidden; + return Name == other.Name && Hidden == other.Hidden && PolicyTag == other.PolicyTag; } } diff --git a/src/Dfe.Analytics.EFCore/DbContextConfigurationExtensions.cs b/src/Dfe.Analytics.EFCore/DbContextConfigurationExtensions.cs index 2343e89..cdf64f1 100644 --- a/src/Dfe.Analytics.EFCore/DbContextConfigurationExtensions.cs +++ b/src/Dfe.Analytics.EFCore/DbContextConfigurationExtensions.cs @@ -5,6 +5,8 @@ namespace Dfe.Analytics.EFCore; public static class DbContextConfigurationExtensions { + private const string DefaultPolicyTagName = "hidden"; + // ReSharper disable once UnusedMethodReturnValue.Global public static T IncludeInAnalyticsSync(this T builder, bool includeAllColumns = true, bool? hidden = null) where T : EntityTypeBuilder @@ -13,19 +15,24 @@ public static T IncludeInAnalyticsSync(this T builder, bool includeAllColumns builder.HasAnnotation( AnnotationKeys.TableAnalyticsSyncMetadata, - new TableSyncMetadata(new ColumnSyncMetadata(includeAllColumns, hidden))); + new TableSyncMetadata(new ColumnSyncMetadata(includeAllColumns, hidden, PolicyTag: DefaultPolicyTagName))); return builder; } - public static T ConfigureAnalyticsSync(this T builder, bool included = true, bool? hidden = null) + public static T ConfigureAnalyticsSync(this T builder, bool included = true, bool? hidden = null, string? policyTag = null) where T : PropertyBuilder { ArgumentNullException.ThrowIfNull(builder); + if (hidden is null && policyTag is not null) + { + hidden = true; + } + builder.HasAnnotation( AnnotationKeys.ColumnAnalyticsSyncMetadata, - new ColumnSyncMetadata(included, hidden)); + new ColumnSyncMetadata(included, hidden, policyTag ?? DefaultPolicyTagName)); return builder; } diff --git a/src/Dfe.Analytics.EFCore/Description/ColumnSyncMetadata.cs b/src/Dfe.Analytics.EFCore/Description/ColumnSyncMetadata.cs index d239592..f24d72e 100644 --- a/src/Dfe.Analytics.EFCore/Description/ColumnSyncMetadata.cs +++ b/src/Dfe.Analytics.EFCore/Description/ColumnSyncMetadata.cs @@ -1,3 +1,3 @@ namespace Dfe.Analytics.EFCore.Description; -internal record ColumnSyncMetadata(bool Included, bool? Hidden); +internal record ColumnSyncMetadata(bool Included, bool? Hidden, string? PolicyTag); diff --git a/tests/Dfe.Analytics.EFCore.Tests/AnalyticsDeployerTests.cs b/tests/Dfe.Analytics.EFCore.Tests/AnalyticsDeployerTests.cs index 65817ba..73e2235 100644 --- a/tests/Dfe.Analytics.EFCore.Tests/AnalyticsDeployerTests.cs +++ b/tests/Dfe.Analytics.EFCore.Tests/AnalyticsDeployerTests.cs @@ -17,6 +17,9 @@ public class AnalyticsDeployerTests private const string DatasetId = "dummy-dataset"; private const string HiddenPolicyTagName = "projects/dummy-project/locations/us/taxonomies/dummy-taxonomy/policyTags/dummy-policy-tag"; + private const string SensitivePolicyTagName = "projects/dummy-project/locations/us/taxonomies/dummy-taxonomy/policyTags/dummy-sensitive-policy-tag"; + + private static readonly IReadOnlyDictionary NoAdditionalPolicyTags = new Dictionary(); [Fact] public async Task ApplyAirbyteConfigurationAsync_UpdatesAirbyteConnectionWithExpectedPayload() @@ -192,7 +195,7 @@ public async Task UpdateBigQueryPolicyTagsAsync_ThrowsIfColumnIsMissingFromBqTab // Act var ex = await Record.ExceptionAsync( - () => deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, progressReporter, TestContext.Current.CancellationToken)); + () => deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, NoAdditionalPolicyTags, progressReporter, TestContext.Current.CancellationToken)); // Assert Assert.IsType(ex); @@ -255,12 +258,128 @@ public async Task UpdateBigQueryPolicyTagsAsync_ColumnIsMissingHiddenPolicyTag_A var deployer = CreateDeployer(httpClient, bigQueryClientMock.Object); // Act - await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, progressReporter, TestContext.Current.CancellationToken); + await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, NoAdditionalPolicyTags, progressReporter, TestContext.Current.CancellationToken); // Assert bigQueryClientMock.Verify(); } + [Fact] + public async Task UpdateBigQueryPolicyTagsAsync_ColumnHasPolicyTag_AddsMappedPolicyTagToSchema() + { + // Arrange + var configuration = GetConfiguration(namePolicyTag: "sensitive"); + var additionalPolicyTags = new Dictionary { ["sensitive"] = SensitivePolicyTagName }; + var progressReporter = new RecordingProgressReporter(); + + var tableName = configuration.Tables.Select(t => t.Name).Single(); + + using var httpClient = new HttpClient(); + + var bigQueryClientMock = new Mock(); + + bigQueryClientMock + .Setup(mock => mock.ListTablesAsync(ProjectId, DatasetId, It.IsAny())) + .Returns(new TestablePagedAsyncEnumerable([ + new BigQueryTable(bigQueryClientMock.Object, new Table { TableReference = new TableReference { TableId = tableName } }) + ])); + + bigQueryClientMock + .Setup(mock => mock.GetTableAsync(ProjectId, DatasetId, tableName, It.IsAny(), It.IsAny())) + .ReturnsAsync(() => + { + var table = new BigQueryTable( + bigQueryClientMock.Object, + new Table + { + Schema = new TableSchema + { + Fields = + [ + new TableFieldSchema { Name = "TestEntityId", Type = "INTEGER" }, + new TableFieldSchema { Name = "Name", Type = "STRING" }, + new TableFieldSchema { Name = "DateOfBirth", Type = "DATE" } + ] + } + }); + + return table; + }); + + bigQueryClientMock + .Setup(mock => mock.PatchTableAsync( + ProjectId, + DatasetId, + tableName, + It.Is(t => + t.Schema.Fields.Single(f => f.Name == "Name").PolicyTags.Names.Contains(SensitivePolicyTagName) && + !t.Schema.Fields.Single(f => f.Name == "Name").PolicyTags.Names.Contains(HiddenPolicyTagName)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => null) + .Verifiable(); + + var deployer = CreateDeployer(httpClient, bigQueryClientMock.Object); + + // Act + await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, additionalPolicyTags, progressReporter, TestContext.Current.CancellationToken); + + // Assert + bigQueryClientMock.Verify(); + } + + [Fact] + public async Task UpdateBigQueryPolicyTagsAsync_ColumnHasPolicyTagWithNoMapping_ThrowsInvalidOperationException() + { + // Arrange + var configuration = GetConfiguration(namePolicyTag: "sensitive"); + var progressReporter = new RecordingProgressReporter(); + + var tableName = configuration.Tables.Select(t => t.Name).Single(); + + using var httpClient = new HttpClient(); + + var bigQueryClientMock = new Mock(); + + bigQueryClientMock + .Setup(mock => mock.ListTablesAsync(ProjectId, DatasetId, It.IsAny())) + .Returns(new TestablePagedAsyncEnumerable([ + new BigQueryTable(bigQueryClientMock.Object, new Table { TableReference = new TableReference { TableId = tableName } }) + ])); + + bigQueryClientMock + .Setup(mock => mock.GetTableAsync(ProjectId, DatasetId, tableName, It.IsAny(), It.IsAny())) + .ReturnsAsync(() => + { + var table = new BigQueryTable( + bigQueryClientMock.Object, + new Table + { + Schema = new TableSchema + { + Fields = + [ + new TableFieldSchema { Name = "TestEntityId", Type = "INTEGER" }, + new TableFieldSchema { Name = "Name", Type = "STRING" }, + new TableFieldSchema { Name = "DateOfBirth", Type = "DATE" } + ] + } + }); + + return table; + }); + + var deployer = CreateDeployer(httpClient, bigQueryClientMock.Object); + + // Act + var ex = await Record.ExceptionAsync( + () => deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, NoAdditionalPolicyTags, progressReporter, TestContext.Current.CancellationToken)); + + // Assert + Assert.IsType(ex); + Assert.Equal("Missing policy tag mapping for 'sensitive'.", ex.Message); + } + [Fact] public async Task UpdateBigQueryPolicyTagsAsync_ColumnHasHiddenPolicyTagButSchemaDoesNotHaveHiddenFlag_RemovesHiddenPolicyTagFromSchema() { @@ -334,7 +453,7 @@ public async Task UpdateBigQueryPolicyTagsAsync_ColumnHasHiddenPolicyTagButSchem var deployer = CreateDeployer(httpClient, bigQueryClientMock.Object); // Act - await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, progressReporter, TestContext.Current.CancellationToken); + await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, NoAdditionalPolicyTags, progressReporter, TestContext.Current.CancellationToken); // Assert bigQueryClientMock.Verify(); @@ -392,7 +511,7 @@ public async Task UpdateBigQueryPolicyTagsAsync_BigQuerySchemaMatchesConfigurati var deployer = CreateDeployer(httpClient, bigQueryClientMock.Object); // Act - await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, progressReporter, TestContext.Current.CancellationToken); + await deployer.UpdateBigQueryPolicyTagsAsync(configuration, HiddenPolicyTagName, NoAdditionalPolicyTags, progressReporter, TestContext.Current.CancellationToken); // Assert bigQueryClientMock.Verify( @@ -478,7 +597,7 @@ private AnalyticsDeployer CreateDeployer( return new AnalyticsDeployer(airbyteApiClient, options); } - private DatabaseSyncConfiguration GetConfiguration() => + private DatabaseSyncConfiguration GetConfiguration(string? namePolicyTag = null) => new() { DbContextName = typeof(TestDbContext).AssemblyQualifiedName!, @@ -490,9 +609,9 @@ private DatabaseSyncConfiguration GetConfiguration() => PrimaryKey = new TablePrimaryKeySyncInfo { ColumnNames = ["TestEntityId"] }, Columns = [ - new ColumnSyncInfo { Name = "TestEntityId", Hidden = false }, - new ColumnSyncInfo { Name = "Name", Hidden = true }, - new ColumnSyncInfo { Name = "DateOfBirth", Hidden = false } + new ColumnSyncInfo { Name = "TestEntityId", Hidden = false, PolicyTag = null }, + new ColumnSyncInfo { Name = "Name", Hidden = true, PolicyTag = namePolicyTag }, + new ColumnSyncInfo { Name = "DateOfBirth", Hidden = false, PolicyTag = null } ] } ] diff --git a/tests/Dfe.Analytics.EFCore.Tests/Configuration/AnalyticsConfigurationProviderTests.cs b/tests/Dfe.Analytics.EFCore.Tests/Configuration/AnalyticsConfigurationProviderTests.cs index 0132cbc..99ccd21 100644 --- a/tests/Dfe.Analytics.EFCore.Tests/Configuration/AnalyticsConfigurationProviderTests.cs +++ b/tests/Dfe.Analytics.EFCore.Tests/Configuration/AnalyticsConfigurationProviderTests.cs @@ -28,21 +28,25 @@ public void GetConfiguration_CreatesValidConfigurationFromDbContext() { Assert.Equal("BaseProperty", column.Name); Assert.False(column.Hidden); + Assert.Null(column.PolicyTag); }, column => { Assert.Equal("DerivedProperty2", column.Name); Assert.False(column.Hidden); + Assert.Null(column.PolicyTag); }, column => { Assert.Equal("Discriminator", column.Name); Assert.False(column.Hidden); + Assert.Null(column.PolicyTag); }, column => { Assert.Equal("Id", column.Name); Assert.False(column.Hidden); + Assert.Null(column.PolicyTag); } ); }, @@ -56,16 +60,27 @@ public void GetConfiguration_CreatesValidConfigurationFromDbContext() { Assert.Equal("DateOfBirth", column.Name); Assert.False(column.Hidden); + Assert.Null(column.PolicyTag); }, column => { + // Specifying a policy tag implies the column is hidden + Assert.Equal("Email", column.Name); + Assert.True(column.Hidden); + Assert.Equal("email", column.PolicyTag); + }, + column => + { + // Hidden columns without an explicit policy tag get the default hidden policy tag Assert.Equal("Name", column.Name); Assert.True(column.Hidden); + Assert.Equal("hidden", column.PolicyTag); }, column => { Assert.Equal("TestEntityId", column.Name); Assert.False(column.Hidden); + Assert.Null(column.PolicyTag); } ); }); diff --git a/tests/Dfe.Analytics.EFCore.Tests/TestDbContext.cs b/tests/Dfe.Analytics.EFCore.Tests/TestDbContext.cs index 0f74c63..9cf6591 100644 --- a/tests/Dfe.Analytics.EFCore.Tests/TestDbContext.cs +++ b/tests/Dfe.Analytics.EFCore.Tests/TestDbContext.cs @@ -15,6 +15,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) testEntityConfiguration.IncludeInAnalyticsSync(hidden: false); testEntityConfiguration.HasKey(t => t.TestEntityId); testEntityConfiguration.Property(t => t.Name).ConfigureAnalyticsSync(hidden: true); + testEntityConfiguration.Property(t => t.Email).ConfigureAnalyticsSync(policyTag: "email"); testEntityConfiguration.Property(t => t.DateOfBirth); testEntityConfiguration.Ignore(t => t.Ignored); diff --git a/tests/Dfe.Analytics.EFCore.Tests/TestEntities.cs b/tests/Dfe.Analytics.EFCore.Tests/TestEntities.cs index f197ada..82ab781 100644 --- a/tests/Dfe.Analytics.EFCore.Tests/TestEntities.cs +++ b/tests/Dfe.Analytics.EFCore.Tests/TestEntities.cs @@ -4,6 +4,7 @@ public class TestEntity { public int TestEntityId { get; set; } public string? Name { get; set; } + public string? Email { get; set; } public string? Ignored { get; set; } public DateOnly DateOfBirth { get; set; } }