WIP: Let validators opt in to run on clean data

Directory.Packages.props

@@ -55,6 +55,7 @@
     <PackageVersion Include="System.Linq.Async" Version="6.0.3" />
     <PackageVersion Include="Testcontainers" Version="4.4.0" />
     <PackageVersion Include="Verify.AspNetCore" Version="4.0.0" />
+    <PackageVersion Include="Verify.Moq" Version="2.2.0" />
     <PackageVersion Include="Verify.Xunit" Version="30.4.0" />
     <PackageVersion Include="Verify.Http" Version="6.6.0" />
     <PackageVersion Include="WireMock.Net" Version="1.8.17" />

src/Altinn.App.Core/Features/IFormDataValidator.cs

@@ -30,13 +30,19 @@ public interface IFormDataValidator
     /// </summary>
     string ValidationSource => $"{this.GetType().FullName}-{DataType}";
 
-    /// <summary>
-    /// If you override this to return true, the validator will only run on process/next, and not continuously.
-    /// <see cref="HasRelevantChanges"/> will never get called
-    /// <see cref="IValidator.NoIncrementalValidation"/>
-    /// </summary>
+    /// <inheritdoc cref="IValidator.NoIncrementalValidation"/>
+    /// <remarks>
+    /// <see cref="IFormDataValidator"/> will run on incremental changes using <see cref="HasRelevantChanges"/>.
+    /// </remarks>
     bool NoIncrementalValidation => false;
 
+    /// <inheritdoc cref="IValidator.ShouldRunAfterRemovingHiddenData"/>
+    /// <remarks>
+    /// Defaults to full data. When <c>true</c>, the pipeline provides a cleaned data accessor to
+    /// both <see cref="ValidateFormData"/> and <see cref="HasRelevantChanges"/> for consistent visibility.
+    /// </remarks>
+    bool ShouldRunAfterRemovingHiddenData => false;
+
     /// <summary>
     /// The actual validation function
     /// </summary>

src/Altinn.App.Core/Features/IInstanceDataAccessor.cs

@@ -1,3 +1,4 @@
+using Altinn.App.Core.Helpers;
 using Altinn.App.Core.Models;
 using Altinn.Platform.Storage.Interface.Models;
 
@@ -32,6 +33,18 @@ public interface IInstanceDataAccessor
     /// <exception cref="InvalidOperationException">when identifier does not exist in instance.Data with an applogic data type</exception>
     Task<IFormDataWrapper> GetFormDataWrapper(DataElementIdentifier dataElementIdentifier);
 
+    /// <summary>
+    /// Get a <see cref="IInstanceDataAccessor"/> that provides access to the cleaned data (where all fields marked as "hidden" are removed).
+    /// </summary>
+    /// <param name="rowRemovalOption">The strategy for "hiddenRow" on group components</param>
+    IInstanceDataAccessor GetCleanAccessor(RowRemovalOption rowRemovalOption = RowRemovalOption.SetToNull);
+
+    /// <summary>
+    /// Get a <see cref="IInstanceDataAccessor"/> that provides access to the
+    /// storage persisted before any in-memory changes in this request.
+    /// </summary>
+    IInstanceDataAccessor GetPreviousDataAccessor();
+
     /// <summary>
     /// Gets the raw binary data from a DataElement.
     /// </summary>

src/Altinn.App.Core/Features/IValidator.cs

@@ -39,6 +39,15 @@ public interface IValidator
     /// </summary>
     bool NoIncrementalValidation => false;
 
+    /// <summary>
+    /// Indicates whether this validator should run against a cleaned view of the data where fields marked as hidden are removed.
+    /// </summary>
+    /// <remarks>
+    /// Defaults to <c>false</c>. When <c>true</c>, the validation pipeline will supply a cleaned accessor for both
+    /// <see cref="Validate"/> and <see cref="HasRelevantChanges"/>, ensuring consistent visibility semantics.
+    /// </remarks>
+    bool ShouldRunAfterRemovingHiddenData => false;
+
     /// <summary>
     /// Run this validator and return all the issues this validator is aware of.
     /// </summary>

src/Altinn.App.Core/Features/Telemetry/Telemetry.Validation.cs

@@ -47,7 +47,8 @@ private static void InitValidation(InitContext context)
         ActivitySource
             .StartActivity($"{Prefix}.RunValidator")
             ?.SetTag(InternalLabels.ValidatorType, validator.GetType().Name)
-            .SetTag(InternalLabels.ValidatorSource, validator.ValidationSource);
+            .SetTag(InternalLabels.ValidatorSource, validator.ValidationSource)
+            .SetTag(InternalLabels.ValidatorRemoveHiddenData, validator.ShouldRunAfterRemovingHiddenData);
 
     internal static class Validation
     {

src/Altinn.App.Core/Features/Telemetry/Telemetry.cs

@@ -228,6 +228,7 @@ internal static class InternalLabels
         internal const string AuthorizerTaskId = "authorization.authorizer.task.id";
         internal const string ValidatorType = "validator.type";
         internal const string ValidatorSource = "validator.source";
+        internal const string ValidatorRemoveHiddenData = "validator.remove_hidden_data";
         internal const string ValidatorHasRelevantChanges = "validator.has_relevant_changes";
         internal const string ValidatorChangedElementsIds = "validator.changed_elements_ids";
         internal const string ValidatorIssueCount = "validation.issue_count";

src/Altinn.App.Core/Features/Validation/Default/LegacyIInstanceValidatorFormDataValidator.cs

@@ -42,7 +42,7 @@ public string ValidationSource
         {
             var type = _instanceValidator?.GetType() ?? GetType();
             Debug.Assert(type.FullName is not null, "FullName does not return null on class/struct types");
-            return type.FullName;
+            return type.FullName + "_FormData";
         }
     }
 

src/Altinn.App.Core/Features/Validation/Default/LegacyIInstanceValidatorTaskValidator.cs

@@ -43,7 +43,7 @@ public string ValidationSource
         {
             var type = _instanceValidator.GetType();
             Debug.Assert(type.FullName is not null, "FullName does not return null on class/struct types");
-            return type.FullName;
+            return type.FullName + "_Task";
         }
     }
 

src/Altinn.App.Core/Features/Validation/GenericFormDataValidator.cs

@@ -32,6 +32,9 @@ protected GenericFormDataValidator(string dataType)
     /// <inheritdoc/>
     public virtual bool NoIncrementalValidation => false;
 
+    /// <inheritdoc />
+    public virtual bool ShouldRunAfterRemovingHiddenData => false;
+
     // ReSharper disable once StaticMemberInGenericType
     private static readonly AsyncLocal<List<ValidationIssue>> _validationIssues = new();
 

src/Altinn.App.Core/Features/Validation/Wrappers/FormDataValidatorWrapper.cs

@@ -35,6 +35,9 @@ IDataElementAccessChecker dataElementAccessChecker
     /// <inheritdoc />
     public bool NoIncrementalValidation => _formDataValidator.NoIncrementalValidation;
 
+    /// <inheritdoc />
+    public bool ShouldRunAfterRemovingHiddenData => _formDataValidator.ShouldRunAfterRemovingHiddenData;
+
     /// <summary>
     /// Run all legacy <see cref="IDataElementValidator"/> instances for the given <see cref="DataType"/>.
     /// </summary>

src/Altinn.App.Core/Internal/Data/CleanInstanceDataAccessor.cs

@@ -0,0 +1,148 @@
+using Altinn.App.Core.Configuration;
+using Altinn.App.Core.Features;
+using Altinn.App.Core.Helpers;
+using Altinn.App.Core.Internal.App;
+using Altinn.App.Core.Internal.Expressions;
+using Altinn.App.Core.Internal.Texts;
+using Altinn.App.Core.Models;
+using Altinn.App.Core.Models.Layout;
+using Altinn.Platform.Storage.Interface.Models;
+
+namespace Altinn.App.Core.Internal.Data;
+
+internal class CleanInstanceDataAccessor : IInstanceDataAccessor
+{
+    private readonly IInstanceDataAccessor _dataAccessor;
+    private readonly string? _taskId;
+    private readonly IAppResources _appResources;
+    private readonly FrontEndSettings _frontEndSettings;
+    private readonly RowRemovalOption _rowRemovalOption;
+    private readonly string? _language;
+    private readonly ITranslationService _translationService;
+    private readonly Telemetry? _telemetry;
+
+    public CleanInstanceDataAccessor(
+        IInstanceDataAccessor dataAccessor,
+        string? taskId,
+        IAppResources appResources,
+        ITranslationService translationService,
+        FrontEndSettings frontEndSettings,
+        RowRemovalOption rowRemovalOption,
+        string? language,
+        Telemetry? telemetry
+    )
+    {
+        _dataAccessor = dataAccessor;
+        _taskId = taskId;
+        _appResources = appResources;
+        _frontEndSettings = frontEndSettings;
+        _rowRemovalOption = rowRemovalOption;
+        _language = language;
+        _telemetry = telemetry;
+        _translationService = translationService;
+
+        LayoutModel? layouts = taskId is not null ? appResources.GetLayoutModelForTask(taskId) : null;
+        if (layouts is null)
+        {
+            _hiddenFieldsTask = new(() => Task.FromResult(new List<DataReference>()));
+        }
+        else
+        {
+            var state = new LayoutEvaluatorState(
+                dataAccessor,
+                layouts,
+                translationService,
+                frontEndSettings,
+                gatewayAction: null,
+                language
+            );
+            _hiddenFieldsTask = new(() =>
+            {
+                using var activity = telemetry?.StartRemoveHiddenDataForValidation();
+                return LayoutEvaluator.GetHiddenFieldsForRemoval(state);
+            });
+        }
+    }
+
+    private readonly DataElementCache<IFormDataWrapper> _cleanCache = new();
+
+    private readonly Lazy<Task<List<DataReference>>> _hiddenFieldsTask;
+
+    public Instance Instance => _dataAccessor.Instance;
+
+    public IReadOnlyCollection<DataType> DataTypes => _dataAccessor.DataTypes;
+
+    public async Task<object> GetFormData(DataElementIdentifier dataElementIdentifier)
+    {
+        return (await GetFormDataWrapper(dataElementIdentifier)).BackingData<object>();
+    }
+
+    public async Task<IFormDataWrapper> GetFormDataWrapper(DataElementIdentifier dataElementIdentifier)
+    {
+        var dataWrapper = await _cleanCache.GetOrCreate(
+            dataElementIdentifier,
+            async () =>
+            {
+                var data = await _dataAccessor.GetFormDataWrapper(dataElementIdentifier).ConfigureAwait(false);
+                var hiddenFields = await _hiddenFieldsTask.Value.ConfigureAwait(false);
+                return CleanModel(data.Copy(), dataElementIdentifier, hiddenFields, _rowRemovalOption);
+            }
+        );
+
+        return dataWrapper.Copy();
+    }
+
+    private static IFormDataWrapper CleanModel(
+        IFormDataWrapper data,
+        DataElementIdentifier dataElementIdentifier,
+        List<DataReference> hiddenFields,
+        RowRemovalOption rowRemovalOption
+    )
+    {
+        foreach (var dataReference in hiddenFields)
+        {
+            if (dataReference.DataElementIdentifier != dataElementIdentifier)
+            {
+                continue;
+            }
+
+            // Note that the paths for lists is in reverse order from GetHiddenFieldsForRemoval, so we can remove them here in order
+            data.RemoveField(dataReference.Field, rowRemovalOption);
+        }
+
+        return data;
+    }
+
+    public IInstanceDataAccessor GetCleanAccessor(RowRemovalOption rowRemovalOption = RowRemovalOption.SetToNull)
+    {
+        if (rowRemovalOption == _rowRemovalOption)
+        {
+            return this;
+        }
+        return new CleanInstanceDataAccessor(
+            _dataAccessor,
+            _taskId,
+            _appResources,
+            _translationService,
+            _frontEndSettings,
+            rowRemovalOption,
+            _language,
+            _telemetry
+        );
+    }
+
+    public IInstanceDataAccessor GetPreviousDataAccessor()
+    {
+        return _dataAccessor.GetPreviousDataAccessor();
+    }
+
+    public async Task<ReadOnlyMemory<byte>> GetBinaryData(DataElementIdentifier dataElementIdentifier)
+    {
+        return await _dataAccessor.GetBinaryData(dataElementIdentifier);
+    }
+
+    public DataElement GetDataElement(DataElementIdentifier dataElementIdentifier)
+    {
+        return _dataAccessor.GetDataElement(dataElementIdentifier);
+    }
+}

src/Altinn.App.Core/Internal/Data/DataElementCache.cs

@@ -21,7 +21,7 @@ public async Task<T> GetOrCreate(DataElementIdentifier key, Func<Task<T>> valueF
                 _cache.Add(key.Guid, lazyTask);
             }
         }
-        return await lazyTask.Value;
+        return await lazyTask.Value.ConfigureAwait(false);
     }
 
     public void Set(DataElementIdentifier key, T data)

src/Altinn.App.Core/Internal/Data/InstanceDataUnitOfWork.cs

@@ -33,10 +33,11 @@ internal sealed class InstanceDataUnitOfWork : IInstanceDataMutator
     private readonly ApplicationMetadata _appMetadata;
     private readonly ModelSerializationService _modelSerializationService;
 
-    // private readonly IAppResources _appResources;
-    // private readonly IOptions<FrontEndSettings> _frontEndSettings;
-    // private readonly string? _taskId;
-    // private readonly string? _language;
+    private readonly IAppResources _appResources;
+    private readonly IOptions<FrontEndSettings> _frontEndSettings;
+    private readonly string? _taskId;
+    private readonly string? _language;
+    private readonly ITranslationService _translationService;
     private readonly Telemetry? _telemetry;
 
     // Cache for the most up-to-date form data (can be mutated or replaced with SetFormData(dataElementId, data))
@@ -51,8 +52,6 @@ internal sealed class InstanceDataUnitOfWork : IInstanceDataMutator
     // Form data not yet saved to storage (thus no dataElementId)
     private readonly ConcurrentBag<DataElementChange> _changesForCreation = [];
 
-    // private readonly ITranslationService _translationService;
-
     public InstanceDataUnitOfWork(
         Instance instance,
         IDataClient dataClient,
@@ -78,12 +77,12 @@ public InstanceDataUnitOfWork(
         DataTypes = appMetadata.DataTypes;
         _dataClient = dataClient;
         _appMetadata = appMetadata;
-        // _translationService = translationService;
+        _translationService = translationService;
         _modelSerializationService = modelSerializationService;
-        // _taskId = taskId;
-        // _language = language;
-        // _frontEndSettings = frontEndSettings;
-        // _appResources = appResources;
+        _taskId = taskId;
+        _language = language;
+        _frontEndSettings = frontEndSettings;
+        _appResources = appResources;
         _instanceClient = instanceClient;
         _telemetry = telemetry;
     }
@@ -121,6 +120,44 @@ public async Task<IFormDataWrapper> GetFormDataWrapper(DataElementIdentifier dat
         );
     }
 
+    /// <inheritdoc />
+    public IInstanceDataAccessor GetCleanAccessor(RowRemovalOption rowRemovalOption = RowRemovalOption.SetToNull)
+    {
+        return new CleanInstanceDataAccessor(
+            this,
+            _taskId,
+            _appResources,
+            _translationService,
+            _frontEndSettings.Value,
+            rowRemovalOption,
+            _language,
+            _telemetry
+        );
+    }
+
+    // Non thread safe cache, because the previous data is always the same.
+    private PreviousDataAccessor? _previousDataAccessorCache;
+
+    public IInstanceDataAccessor GetPreviousDataAccessor()
+    {
+        if (_previousDataAccessorCache is not null)
+        {
+            return _previousDataAccessorCache;
+        }
+
+        _previousDataAccessorCache = new PreviousDataAccessor(
+            this,
+            _taskId,
+            _appResources,
+            _translationService,
+            _modelSerializationService,
+            _frontEndSettings.Value,
+            _language,
+            _telemetry
+        );
+        return _previousDataAccessorCache;
+    }
+
     /// <inheritdoc />
     public async Task<ReadOnlyMemory<byte>> GetBinaryData(DataElementIdentifier dataElementIdentifier)
     {