[WIP] Feat/remove when hidden

src/Altinn.App.Analyzers/IncrementalGenerator/FormDataWrapperGenerator.cs → src/Altinn.App.Analyzers/FormDataWrapperGenerator.cs

@@ -2,7 +2,7 @@
 using Altinn.App.Analyzers.Utils;
 using NanoJsonReader;
 
-namespace Altinn.App.Analyzers.IncrementalGenerator;
+namespace Altinn.App.Analyzers;
 
 /// <summary>
 /// Generate IFormDataWrapper implementations for classes in models/*.cs in the app.
@@ -249,6 +249,6 @@ private static void GenerateFromNode(SourceProductionContext context, Result<Mod
             return;
 
         var sourceText = SourceTextGenerator.SourceTextGenerator.GenerateSourceText(node);
-        context.AddSource(node.Name + "FormDataWrapper.g.cs", sourceText);
+        context.AddSource(node.Name.Split('_').Last() + "FormDataWrapper.g.cs", sourceText);
     }
 }

src/Altinn.App.Core/Features/Validation/Default/ExpressionValidator.cs

@@ -119,7 +119,10 @@ internal async Task<List<ValidationIssue>> ValidateFormData(
             gatewayAction: null,
             language
         );
-        var hiddenFields = await LayoutEvaluator.GetHiddenFieldsForRemoval(evaluatorState);
+        var hiddenFields = await LayoutEvaluator.GetHiddenFieldsForRemoval(
+            evaluatorState,
+            evaluateRemoveWhenHidden: false
+        );
 
         var validationIssues = new List<ValidationIssue>();
         DataElementIdentifier dataElementIdentifier = dataElement;

src/Altinn.App.Core/Internal/Data/CleanInstanceDataAccessor.cs

@@ -59,7 +59,7 @@ public CleanInstanceDataAccessor(
             _hiddenFieldsTask = new(() =>
             {
                 using var activity = telemetry?.StartRemoveHiddenDataForValidation();
-                return LayoutEvaluator.GetHiddenFieldsForRemoval(state);
+                return LayoutEvaluator.GetHiddenFieldsForRemoval(state, evaluateRemoveWhenHidden: false);
             });
         }
     }

src/Altinn.App.Core/Internal/Expressions/ExpressionEvaluator.cs

@@ -29,6 +29,7 @@ bool defaultReturn
             {
                 "hidden" => context.Component.Hidden,
                 "required" => context.Component.Required,
+                "removeWhenHidden" => context.Component.RemoveWhenHidden,
                 _ => throw new ExpressionEvaluatorTypeErrorException($"unknown boolean expression property {property}"),
             };
 
@@ -268,7 +269,7 @@ LayoutEvaluatorState state
         {
             throw new ArgumentException("component lookup requires the target component to have a simpleBinding");
         }
-        if (await targetContext.IsHidden())
+        if (await targetContext.IsHidden(evaluateRemoveWhenHidden: false))
         {
             return ExpressionValue.Null;
         }

src/Altinn.App.Core/Internal/Expressions/LayoutEvaluator.cs

@@ -13,15 +13,32 @@ public static class LayoutEvaluator
     /// <summary>
     /// Get a list of fields that are only referenced in hidden components in <see cref="LayoutEvaluatorState" />
     /// </summary>
-    public static async Task<List<DataReference>> GetHiddenFieldsForRemoval(LayoutEvaluatorState state)
+    [Obsolete("Use the overload with evaluateRemoveWhenHidden parameter")]
+    public static async Task<List<DataReference>> GetHiddenFieldsForRemoval(LayoutEvaluatorState state) =>
+        await GetHiddenFieldsForRemoval(state, evaluateRemoveWhenHidden: false);
+
+    /// <summary>
+    /// Get a list of fields that are only referenced in hidden components in <see cref="LayoutEvaluatorState" />
+    /// </summary>
+    public static async Task<List<DataReference>> GetHiddenFieldsForRemoval(
+        LayoutEvaluatorState state,
+        bool evaluateRemoveWhenHidden
+    )
     {
         var hiddenModelBindings = new HashSet<DataReference>();
         var nonHiddenModelBindings = new HashSet<DataReference>();
 
         var pageContexts = await state.GetComponentContexts();
         foreach (var pageContext in pageContexts)
         {
-            await HiddenFieldsForRemovalRecurs(state, hiddenModelBindings, nonHiddenModelBindings, pageContext, []);
+            await HiddenFieldsForRemovalRecurs(
+                state,
+                hiddenModelBindings,
+                nonHiddenModelBindings,
+                pageContext,
+                evaluateRemoveWhenHidden,
+                []
+            );
         }
 
         var forRemoval = hiddenModelBindings.Except(nonHiddenModelBindings).ToList();
@@ -34,6 +51,7 @@ private static async Task HiddenFieldsForRemovalRecurs(
         HashSet<DataReference> hiddenModelBindings,
         HashSet<DataReference> nonHiddenModelBindings,
         ComponentContext context,
+        bool evaluateRemoveWhenHidden,
         IReadOnlyList<DataReference> ignoredPrefixes
     )
     {
@@ -45,7 +63,7 @@ IReadOnlyList<DataReference> ignoredPrefixes
             );
         }
 
-        var isHidden = await context.IsHidden();
+        var isHidden = await context.IsHidden(evaluateRemoveWhenHidden);
 
         List<DataReference> childIgnoredPrefixes = [.. ignoredPrefixes];
 
@@ -77,6 +95,7 @@ await HiddenFieldsForRemovalRecurs(
                 hiddenModelBindings,
                 nonHiddenModelBindings,
                 childContext,
+                evaluateRemoveWhenHidden,
                 childIgnoredPrefixes
             );
         }
@@ -88,15 +107,26 @@ await HiddenFieldsForRemovalRecurs(
     [Obsolete("Use the async version of this method RemoveHiddenDataAsync")]
     public static void RemoveHiddenData(LayoutEvaluatorState state, RowRemovalOption rowRemovalOption)
     {
-        RemoveHiddenDataAsync(state, rowRemovalOption).GetAwaiter().GetResult();
+        RemoveHiddenDataAsync(state, rowRemovalOption, evaluateRemoveWhenHidden: false).GetAwaiter().GetResult();
     }
 
     /// <summary>
     /// Remove fields that are only referenced from hidden fields from the data object in the state.
     /// </summary>
-    public static async Task RemoveHiddenDataAsync(LayoutEvaluatorState state, RowRemovalOption rowRemovalOption)
+    [Obsolete("Use the overload with evaluateRemoveWhenHidden parameter")]
+    public static async Task RemoveHiddenDataAsync(LayoutEvaluatorState state, RowRemovalOption rowRemovalOption) =>
+        await RemoveHiddenDataAsync(state, rowRemovalOption, evaluateRemoveWhenHidden: false);
+
+    /// <summary>
+    /// Remove fields that are only referenced from hidden fields from the data object in the state.
+    /// </summary>
+    public static async Task RemoveHiddenDataAsync(
+        LayoutEvaluatorState state,
+        RowRemovalOption rowRemovalOption,
+        bool evaluateRemoveWhenHidden
+    )
     {
-        var fields = await GetHiddenFieldsForRemoval(state);
+        var fields = await GetHiddenFieldsForRemoval(state, evaluateRemoveWhenHidden);
 
         // Ensure fields with higher row numbers are removed before fields with lower row numbers.
         foreach (var dataReference in OrderByListIndexReverse(fields))
@@ -127,7 +157,7 @@ ComponentContext context
     )
     {
         ArgumentNullException.ThrowIfNull(context.Component);
-        var hidden = await context.IsHidden();
+        var hidden = await context.IsHidden(evaluateRemoveWhenHidden: false);
         if (!hidden)
         {
             foreach (var childContext in context.ChildContexts)

src/Altinn.App.Core/Internal/Process/ProcessTasks/Common/ProcessTaskFinalizer.cs

@@ -101,7 +101,11 @@ private async Task RemoveFieldsOnTaskComplete(
                 gatewayAction: null,
                 language
             );
-            await LayoutEvaluator.RemoveHiddenDataAsync(evaluationState, RowRemovalOption.DeleteRow);
+            await LayoutEvaluator.RemoveHiddenDataAsync(
+                evaluationState,
+                RowRemovalOption.DeleteRow,
+                evaluateRemoveWhenHidden: true
+            );
         }
 
         // Remove shadow fields

src/Altinn.App.Core/Models/Expressions/ComponentContext.cs

@@ -53,20 +53,44 @@ public ComponentContext(
     /// </summary>
     public int[]? RowIndices { get; }
 
+    /// <summary>
+    /// Memoization for evaluation of hidden
+    /// </summary>
     private bool? _isHidden;
 
+    /// <summary>
+    /// Memoization for evaluation of removeWhenHidden
+    /// </summary>
+    private bool? _removeWhenHidden;
+
     /// <summary>
     /// Memoized way to check if the component is hidden
     /// </summary>
-    public async Task<bool> IsHidden()
+    public async Task<bool> IsHidden(bool evaluateRemoveWhenHidden)
     {
-        if (_isHidden.HasValue)
+        if (evaluateRemoveWhenHidden)
         {
-            return _isHidden.Value;
+            // We will check parent removeWhenHidden when we check parent hidden
+            var removeWhenHidden = await GetMemoizedRemoveWhenHidden();
+            if (!removeWhenHidden)
+            {
+                return false;
+            }
+        }
+
+        // If the parent is hidden, this is also hidden
+        if (Parent is not null && await Parent.IsHidden(evaluateRemoveWhenHidden))
+        {
+            return true;
         }
-        if (Parent is not null && await Parent.IsHidden())
+
+        return await GetMemoizedIsHidden();
+    }
+
+    private async Task<bool> GetMemoizedIsHidden()
+    {
+        if (_isHidden.HasValue)
         {
-            _isHidden = true;
             return _isHidden.Value;
         }
 
@@ -86,12 +110,31 @@ public async Task<bool> IsHidden()
             }
         }
 
-        var hidden = await ExpressionEvaluator.EvaluateBooleanExpression(State, this, "hidden", false);
-
-        _isHidden = hidden;
+        var isHidden = await ExpressionEvaluator.EvaluateBooleanExpression(State, this, "hidden", false);
+        _isHidden = isHidden;
         return _isHidden.Value;
     }
 
+    private async Task<bool> GetMemoizedRemoveWhenHidden()
+    {
+        if (_removeWhenHidden.HasValue)
+        {
+            return _removeWhenHidden.Value;
+        }
+
+        var removeWhenHidden = await ExpressionEvaluator.EvaluateBooleanExpression(
+            State,
+            this,
+            "removeWhenHidden",
+            // Default return should match AppSettings.RemoveHiddenData,
+            // but currently we only run removal when it is true, so we set it to true here
+            defaultReturn: true
+        );
+
+        _removeWhenHidden = removeWhenHidden;
+        return _removeWhenHidden.Value;
+    }
+
     /// <summary>
     /// Indicates whether this context was initialized with child contexts
     /// </summary>

test/Altinn.App.Api.Tests/Controllers/DataController_LayoutEvaluatorTests.cs

@@ -49,7 +49,12 @@ public async Task ProcessDataWrite(
                 var id = new DataElementIdentifier(dataId.Value);
                 hidden
                     .Should()
-                    .BeEquivalentTo([new DataReference() { DataElementIdentifier = id, Field = "melding.hidden" }]);
+                    .BeEquivalentTo(
+                        [
+                            new DataReference() { DataElementIdentifier = id, Field = "melding.hidden" },
+                            new DataReference() { DataElementIdentifier = id, Field = "melding.hiddenNotRemove" },
+                        ]
+                    );
                 if (data is Skjema { Melding: { } melding })
                 {
                     melding.Toggle = !melding.Toggle;

test/Altinn.App.Api.Tests/Controllers/ProcessControllerTests.cs

@@ -422,6 +422,10 @@ public async Task RunProcessNext_DataFromHiddenComponents_GetsRemoved()
                     PatchOperation.Add(
                         JsonPointer.Create("melding", "hidden"),
                         JsonNode.Parse("\"value that is hidden\"")
+                    ),
+                    PatchOperation.Add(
+                        JsonPointer.Create("melding", "hiddenNotRemove"),
+                        JsonNode.Parse("\"value that is not removed\"")
                     )
                 ),
                 IgnoredValidators = [],
@@ -441,6 +445,7 @@ public async Task RunProcessNext_DataFromHiddenComponents_GetsRemoved()
         OutputHelper.WriteLine("Data before process next:");
         OutputHelper.WriteLine(dataString);
         dataString.Should().Contain("<hidden>value that is hidden</hidden>");
+        dataString.Should().Contain("<hiddenNotRemove>value that is not removed</hiddenNotRemove>");
 
         // Run process next
         var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", null);
@@ -453,6 +458,7 @@ public async Task RunProcessNext_DataFromHiddenComponents_GetsRemoved()
         OutputHelper.WriteLine("Data after process next:");
         OutputHelper.WriteLine(dataString);
         dataString.Should().NotContain("<hidden>value that is hidden</hidden>");
+        dataString.Should().Contain("<hiddenNotRemove>value that is not removed</hiddenNotRemove>");
 
         _dataProcessorMock.Verify();
     }

test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/models/Skjema.cs

@@ -66,6 +66,11 @@ public bool ShouldSerializeTagWithAttribute()
     [JsonProperty("SF_test")]
     [JsonPropertyName("SF_test")]
     public string? SF_test { get; set; }
+
+    [XmlElement("hiddenNotRemove", Order = 10)]
+    [JsonProperty("hiddenNotRemove")]
+    [JsonPropertyName("hiddenNotRemove")]
+    public string? HiddenNotRemove { get; set; }
 }
 
 public class TagWithAttribute

test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/ui/default/layouts/page.json

@@ -25,6 +25,15 @@
         "dataModelBindings": {
           "simpleBinding": "melding.hidden"
         }
+      },
+      {
+        "id": "hiddenNotRemove",
+        "type": "Input",
+        "hidden": true,
+        "removeWhenHidden": false,
+        "dataModelBindings": {
+          "simpleBinding": "melding.hiddenNotRemove"
+        }
       }
     ]
   }

test/Altinn.App.Api.Tests/OpenApi/OpenApiSpecChangeDetection.SaveCustomOpenApiSpec.verified.json

@@ -3258,6 +3258,10 @@
           "SF_test": {
             "type": "string",
             "nullable": true
+          },
+          "hiddenNotRemove": {
+            "type": "string",
+            "nullable": true
           }
         },
         "additionalProperties": false

test/Altinn.App.Core.Tests/LayoutExpressions/FullTests/RemoveHiddenData/RemoveHiddenDataTests.cs

@@ -11,7 +11,8 @@ public class RemoveHiddenDataTests
     [Fact]
     public async Task TestRemoveHiddenData()
     {
-        var jsonData = """
+        using var jsonDoc = JsonDocument.Parse(
+            """
             {
                 "root": {
                     "fornavn": null,
@@ -107,8 +108,8 @@ public async Task TestRemoveHiddenData()
                     "vedlegg": []
                 }
             }
-            """;
-        using var jsonDoc = JsonDocument.Parse(jsonData);
+            """
+        );
         IInstanceDataAccessor dataAccessor = DynamicClassBuilder.DataAccessorFromJsonDocument(
             new Instance(),
             jsonDoc.RootElement

test/Altinn.App.Core.Tests/LayoutExpressions/FullTests/SubForm/SubFormTests.Test1.verified.txt

@@ -116,4 +116,4 @@
     Source: Required,
     NoIncrementalUpdates: false
   }
-]
+]

test/Altinn.App.Core.Tests/PublicApiTests.PublicApi_ShouldNotChange_Unintentionally.verified.txt

@@ -3058,10 +3058,14 @@ namespace Altinn.App.Core.Internal.Expressions
     }
     public static class LayoutEvaluator
     {
+        [System.Obsolete("Use the overload with evaluateRemoveWhenHidden parameter")]
         public static System.Threading.Tasks.Task<System.Collections.Generic.List<Altinn.App.Core.Models.Layout.DataReference>> GetHiddenFieldsForRemoval(Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState state) { }
+        public static System.Threading.Tasks.Task<System.Collections.Generic.List<Altinn.App.Core.Models.Layout.DataReference>> GetHiddenFieldsForRemoval(Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState state, bool evaluateRemoveWhenHidden) { }
         [System.Obsolete("Use the async version of this method RemoveHiddenDataAsync")]
         public static void RemoveHiddenData(Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState state, Altinn.App.Core.Helpers.RowRemovalOption rowRemovalOption) { }
+        [System.Obsolete("Use the overload with evaluateRemoveWhenHidden parameter")]
         public static System.Threading.Tasks.Task RemoveHiddenDataAsync(Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState state, Altinn.App.Core.Helpers.RowRemovalOption rowRemovalOption) { }
+        public static System.Threading.Tasks.Task RemoveHiddenDataAsync(Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState state, Altinn.App.Core.Helpers.RowRemovalOption rowRemovalOption, bool evaluateRemoveWhenHidden) { }
         public static System.Threading.Tasks.Task<System.Collections.Generic.List<Altinn.App.Core.Models.Validation.ValidationIssue>> RunLayoutValidationsForRequired(Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState state) { }
     }
     public class LayoutEvaluatorState
@@ -4276,7 +4280,7 @@ namespace Altinn.App.Core.Models.Expressions
         public Altinn.App.Core.Models.Expressions.ComponentContext? Parent { get; }
         public int[]? RowIndices { get; }
         public Altinn.App.Core.Internal.Expressions.LayoutEvaluatorState State { get; }
-        public System.Threading.Tasks.Task<bool> IsHidden() { }
+        public System.Threading.Tasks.Task<bool> IsHidden(bool evaluateRemoveWhenHidden) { }
     }
     [System.Text.Json.Serialization.JsonConverter(typeof(Altinn.App.Core.Models.Expressions.ExpressionConverter?))]
     public readonly struct Expression : System.IEquatable<Altinn.App.Core.Models.Expressions.Expression>

test/Altinn.App.SourceGenerator.Integration.Tests/Altinn.App.SourceGenerator.Integration.Tests.csproj

@@ -7,7 +7,7 @@
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
     <EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
-    <CompilerGeneratedFilesOutputPath>Generated</CompilerGeneratedFilesOutputPath>
+    <CompilerGeneratedFilesOutputPath>gen</CompilerGeneratedFilesOutputPath>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="coverlet.collector" />