chore: Remove confidential values [DHIS2-21518]

dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java

@@ -459,28 +459,22 @@ public List<TrackedEntityAttribute> getTrackedEntityAttributes() {
   }
 
   /**
-   * Returns non-confidential TrackedEntityAttributes from ProgramTrackedEntityAttributes. Use
-   * getAttributes() to access the persisted attribute list. Skipped attributes are also considered
-   * confidential.
+   * Returns non-skipped TrackedEntityAttributes from ProgramTrackedEntityAttributes. Use
+   * getAttributes() to access the persisted attribute list.
    */
-  public List<TrackedEntityAttribute> getNonConfidentialTrackedEntityAttributes() {
+  public List<TrackedEntityAttribute> getAnalyzableTrackedEntityAttributes() {
     return getTrackedEntityAttributes().stream()
-        .filter(a -> !a.isConfidentialBool() && !a.isSkipAnalytics())
+        .filter(a -> !a.getSkipAnalytics())
         .collect(Collectors.toList());
   }
 
   /**
-   * Returns TrackedEntityAttributes from ProgramTrackedEntityAttributes which have a legend set and
-   * is of numeric value type. Skipped attributes are also considered confidential.
+   * Returns TrackedEntityAttributes from ProgramTrackedEntityAttributes which have a legend set,
+   * are of numeric value type and are not skipped.
    */
-  public List<TrackedEntityAttribute> getNonConfidentialTrackedEntityAttributesWithLegendSet() {
+  public List<TrackedEntityAttribute> getAnalyzableTrackedEntityAttributesWithLegendSet() {
     return getTrackedEntityAttributes().stream()
-        .filter(
-            a ->
-                !a.isConfidentialBool()
-                    && !a.isSkipAnalytics()
-                    && a.hasLegendSet()
-                    && a.isNumericType())
+        .filter(a -> !a.getSkipAnalytics() && a.hasLegendSet() && a.isNumericType())
         .collect(Collectors.toList());
   }
 

dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemInfo.java

@@ -96,7 +96,6 @@ public final class SystemInfo {
   @JsonProperty private final Integer readReplicaCount;
   @JsonProperty private final String memoryInfo;
   @JsonProperty private final Integer cpuCores;
-  @JsonProperty private final boolean encryption;
   @JsonProperty private final boolean emailConfigured;
   @JsonProperty private final boolean redisEnabled;
   @JsonProperty private final String redisHostname;
@@ -109,6 +108,7 @@ public final class SystemInfo {
   @JsonProperty private final Boolean isMetadataVersionEnabled;
   @JsonProperty private final Date lastMetadataVersionSyncAttempt;
   @JsonProperty private final Boolean isMetadataSyncEnabled;
+  @JsonProperty private final boolean encryption;
 
   /**
    * Clears sensitive system info properties.
@@ -134,10 +134,10 @@ public SystemInfo withoutSensitiveInfo() {
         .memoryInfo(null)
         .cpuCores(null)
         .systemMonitoringUrl(null)
-        .encryption(false)
         .redisEnabled(false)
         .redisHostname(null)
         .clusterHostname(null)
+        .encryption(false)
         .databaseInfo(databaseInfo.withoutSensitiveInfo())
         .build();
   }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/trackedentity/TrackedEntityAttribute.java

@@ -77,8 +77,6 @@ public class TrackedEntityAttribute extends BaseDimensionalItemObject
 
   private Integer sortOrderInListNoProgram;
 
-  private Boolean confidential = false;
-
   private Boolean unique = false;
 
   // For TextPattern:
@@ -120,7 +118,7 @@ public class TrackedEntityAttribute extends BaseDimensionalItemObject
   @Transient private Boolean trigramIndexed = false;
 
   @OpenApi.Description("Indicates whether this attribute should be excluded from analytics.")
-  private boolean skipAnalytics;
+  private Boolean skipAnalytics = false;
 
   // -------------------------------------------------------------------------
   // Constructors
@@ -155,12 +153,6 @@ public boolean isDateType() {
     return valueType.isDate();
   }
 
-  /** Indicates whether this attribute has confidential information. */
-  @JsonIgnore
-  public boolean isConfidentialBool() {
-    return confidential != null && confidential;
-  }
-
   /** Indicates whether this attribute has an option set. */
   @Override
   public boolean hasOptionSet() {
@@ -342,16 +334,6 @@ public void setOptionSet(OptionSet optionSet) {
     this.optionSet = optionSet;
   }
 
-  @JsonProperty
-  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
-  public Boolean getConfidential() {
-    return confidential;
-  }
-
-  public void setConfidential(Boolean confidential) {
-    this.confidential = confidential;
-  }
-
   public TextPattern getTextPattern() {
     return textPattern;
   }
@@ -454,11 +436,11 @@ public void setTrigramIndexed(Boolean trigramIndexed) {
 
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
-  public boolean isSkipAnalytics() {
+  public Boolean getSkipAnalytics() {
     return skipAnalytics;
   }
 
-  public void setSkipAnalytics(boolean skipAnalytics) {
+  public void setSkipAnalytics(Boolean skipAnalytics) {
     this.skipAnalytics = skipAnalytics;
   }
 
@@ -488,8 +470,6 @@ public String toString() {
         + displayInListNoProgram
         + ", sortOrderInListNoProgram="
         + sortOrderInListNoProgram
-        + ", confidential="
-        + confidential
         + ", unique="
         + unique
         + ", generated="

dhis-2/dhis-api/src/main/java/org/hisp/dhis/tracker/imports/validation/ValidationCode.java

@@ -112,7 +112,6 @@ public enum ValidationCode {
   E1103("User: `{0}` lacks 'F_ENROLLMENT_CASCADE_DELETE' authority to delete Enrollment : `{1}`."),
   E1104("User: `{0}` has no data read access to Program: `{1}`, TrackedEntityType: `{2}`."),
   E1105("User: `{0}` has no search scope access to OrganisationUnit: `{1}`."),
-  E1112("AttributeValue: `{0}` is confidential but encryption is not enabled: `{1}`"),
   E1113("Enrollment: `{0}` is already deleted and cannot be modified."),
   E1114("TrackedEntity: `{0}` is already deleted and cannot be modified."),
   E1115("Could not find CategoryOptionCombo: `{0}`."),

dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java

@@ -188,8 +188,8 @@ void testCopyOfWithNulls() {
     assertTrue(copy.getAnalyticsDataElements().isEmpty());
     assertTrue(copy.getCategoryMappings().isEmpty());
     assertTrue(copy.getDataElements().isEmpty());
-    assertTrue(copy.getNonConfidentialTrackedEntityAttributes().isEmpty());
-    assertTrue(copy.getNonConfidentialTrackedEntityAttributesWithLegendSet().isEmpty());
+    assertTrue(copy.getAnalyzableTrackedEntityAttributes().isEmpty());
+    assertTrue(copy.getAnalyzableTrackedEntityAttributesWithLegendSet().isEmpty());
     assertTrue(copy.getNotificationTemplates().isEmpty());
     assertTrue(copy.getOrganisationUnits().isEmpty());
     assertTrue(copy.getProgramAttributes().isEmpty());

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java

@@ -2500,7 +2500,7 @@ private Set<String> getEnrollmentColumnsFromProgramIndicators(EventQueryParams p
     Set<String> enrollmentColumns = new HashSet<>();
 
     Set<String> columns =
-        params.getProgram().getNonConfidentialTrackedEntityAttributes().stream()
+        params.getProgram().getAnalyzableTrackedEntityAttributes().stream()
             .map(columnMapper::getColumnsForAttribute)
             .flatMap(Collection::stream)
             .map(AnalyticsTableColumn::getName)

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java

@@ -81,7 +81,7 @@ public List<PrefixedDimension> getQueryDimensionsByProgramId(String programId) {
                         filterByValueType(
                             QUERY,
                             ofItemsWithProgram(
-                                program, getTeasIfRegistrationAndNotConfidential(program))))))
+                                program, getTeasIfRegistrationAndNotSkipped(program))))))
         .orElse(List.of());
   }
 
@@ -127,23 +127,17 @@ public List<PrefixedDimension> getAggregateDimensionsByProgramId(String programI
         .orElse(List.of());
   }
 
-  private Collection<TrackedEntityAttribute> getTeasIfRegistrationAndNotConfidential(
-      Program program) {
+  private Collection<TrackedEntityAttribute> getTeasIfRegistrationAndNotSkipped(Program program) {
     return Optional.of(program)
         .filter(Program::isRegistration)
         .map(Program::getTrackedEntityAttributes)
         .orElse(List.of())
         .stream()
-        .filter(this::isNotConfidential)
         .filter(this::isNotSkipped)
         .collect(Collectors.toList());
   }
 
-  private boolean isNotConfidential(TrackedEntityAttribute trackedEntityAttribute) {
-    return !trackedEntityAttribute.isConfidentialBool();
-  }
-
   private boolean isNotSkipped(TrackedEntityAttribute trackedEntityAttribute) {
-    return !trackedEntityAttribute.isSkipAnalytics();
+    return !Boolean.TRUE.equals(trackedEntityAttribute.getSkipAnalytics());
   }
 }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java

@@ -158,8 +158,7 @@ private List<PrefixedDimension> dimensions(ProgramStage programStage) {
                                 .collect(Collectors.toSet())),
                         filterByValueType(QUERY, ofDataElements(programStage)),
                         filterByValueType(
-                            QUERY,
-                            ofItemsWithProgram(p, getTeasIfRegistrationAndNotConfidential(p))),
+                            QUERY, ofItemsWithProgram(p, getTeasIfRegistrationAndAnalyzable(p))),
                         ofItemsWithProgram(p, getCategories(p)),
                         ofItemsWithProgram(p, getAttributeCategoryOptionGroupSetsIfNeeded(p)))))
         .orElse(List.of());
@@ -189,22 +188,17 @@ private List<Category> getCategories(Program program) {
         .orElse(List.of());
   }
 
-  private List<TrackedEntityAttribute> getTeasIfRegistrationAndNotConfidential(Program program) {
+  private List<TrackedEntityAttribute> getTeasIfRegistrationAndAnalyzable(Program program) {
     return Optional.of(program)
         .filter(Program::isRegistration)
         .map(Program::getTrackedEntityAttributes)
         .orElse(List.of())
         .stream()
-        .filter(this::isNotConfidential)
         .filter(this::isNotSkipped)
         .collect(Collectors.toList());
   }
 
-  private boolean isNotConfidential(TrackedEntityAttribute trackedEntityAttribute) {
-    return !trackedEntityAttribute.isConfidentialBool();
-  }
-
   private boolean isNotSkipped(TrackedEntityAttribute trackedEntityAttribute) {
-    return !trackedEntityAttribute.isSkipAnalytics();
+    return !trackedEntityAttribute.getSkipAnalytics();
   }
 }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java

@@ -143,7 +143,7 @@ protected String getAttributeValueJoinClause(Program program) {
         on en.trackedentityid=${uid}.trackedentityid \
         and ${uid}.trackedentityattributeid = ${id}\s""";
 
-    return program.getNonConfidentialTrackedEntityAttributes().stream()
+    return program.getAnalyzableTrackedEntityAttributes().stream()
         .map(attribute -> replaceQualify(sqlBuilder, template, toVariableMap(attribute)))
         .collect(Collectors.joining());
   }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java

@@ -205,7 +205,7 @@ private List<AnalyticsTableColumn> getColumns(Program program) {
    * @return a list of {@link AnalyticsTableColumn}.
    */
   private List<AnalyticsTableColumn> getTrackedEntityAttributeColumns(Program program) {
-    return program.getNonConfidentialTrackedEntityAttributes().stream()
+    return program.getAnalyzableTrackedEntityAttributes().stream()
         .map(this::getColumnForAttribute)
         .flatMap(Collection::stream)
         .toList();

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java

@@ -752,12 +752,12 @@ private String getSelectExpression(DataElement dataElement, String columnExpress
   private List<AnalyticsTableColumn> getAttributeColumns(Program program) {
     List<AnalyticsTableColumn> columns = new ArrayList<>();
     columns.addAll(
-        program.getNonConfidentialTrackedEntityAttributes().stream()
+        program.getAnalyzableTrackedEntityAttributes().stream()
             .map(this::getColumnForAttribute)
             .flatMap(Collection::stream)
             .toList());
     columns.addAll(
-        program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream()
+        program.getAnalyzableTrackedEntityAttributesWithLegendSet().stream()
             .map(this::getColumnForAttributeWithLegendSet)
             .flatMap(Collection::stream)
             .toList());

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java

@@ -219,7 +219,7 @@ private List<AnalyticsTableColumn> getColumns(
 
     List<TrackedEntityAttribute> trackedEntityAttributes =
         getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid)
-            .filter(tea -> !tea.isConfidentialBool() && !tea.isSkipAnalytics())
+            .filter(tea -> !tea.getSkipAnalytics())
             .toList();
 
     params.addExtraParam(
@@ -334,7 +334,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti
         ((List<TrackedEntityAttribute>)
                 params.getExtraParam(
                     trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES))
-            .stream().filter(tea -> !tea.isSkipAnalytics()).toList();
+            .stream().filter(tea -> !tea.getSkipAnalytics()).toList();
 
     if (isNotEmpty(attributes)) {
       attributes.forEach(

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java

@@ -126,54 +126,37 @@ void setUp() {
   }
 
   @Test
-  void verifyNonConfidentialTeasAreSkipped() {
+  void verifySkippedTeasAreExcluded() {
     AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder().build();
 
     TrackedEntityType tet = mock(TrackedEntityType.class);
     when(tet.getUid()).thenReturn("tetUid");
 
-    TrackedEntityAttribute nonConfidentialTea = new TrackedEntityAttribute();
-    nonConfidentialTea.setUid("nonConfidentialTeaUid");
-    nonConfidentialTea.setConfidential(false);
-    nonConfidentialTea.setValueType(ValueType.TEXT);
+    TrackedEntityAttribute tea = new TrackedEntityAttribute();
+    tea.setUid("teaUid");
+    tea.setSkipAnalytics(false);
+    tea.setValueType(ValueType.TEXT);
 
-    TrackedEntityAttribute confidentialTea = new TrackedEntityAttribute();
-    confidentialTea.setUid("confidentialTeaUid");
-    confidentialTea.setConfidential(true);
-    confidentialTea.setValueType(ValueType.TEXT);
+    TrackedEntityAttribute skippedTea = new TrackedEntityAttribute();
+    skippedTea.setUid("skippedTeaUid");
+    skippedTea.setSkipAnalytics(true);
+    skippedTea.setValueType(ValueType.TEXT);
 
     Program program = mock(Program.class);
 
-    when(tet.getTrackedEntityAttributes()).thenReturn(List.of(nonConfidentialTea, confidentialTea));
-
+    when(tet.getTrackedEntityAttributes()).thenReturn(List.of(tea, skippedTea));
     when(program.getTrackedEntityType()).thenReturn(tet);
-
     when(trackedEntityTypeService.getAllTrackedEntityType()).thenReturn(List.of(tet));
-
     when(trackedEntityAttributeService.getProgramTrackedEntityAttributes(List.of(program)))
         .thenReturn(List.of());
-
     when(identifiableObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program));
 
     List<AnalyticsTable> analyticsTables = tableManager.getAnalyticsTables(params);
 
     assertEquals(1, analyticsTables.size());
 
-    AnalyticsTable analyticsTable = analyticsTables.get(0);
-
-    assertContainsNonConfidentialTeaColumns(analyticsTable);
-    assertDoesntContainConfidentialTeaColumns(analyticsTable);
-  }
-
-  private void assertDoesntContainConfidentialTeaColumns(AnalyticsTable analyticsTable) {
-    List<Column> columns = analyticsTable.getColumns();
-
-    assertFalse(columns.stream().map(Column::getName).anyMatch("confidentialTeaUid"::equals));
-  }
-
-  private void assertContainsNonConfidentialTeaColumns(AnalyticsTable analyticsTable) {
-    List<Column> columns = analyticsTable.getColumns();
-
-    assertTrue(columns.stream().map(Column::getName).anyMatch("nonConfidentialTeaUid"::equals));
+    List<Column> columns = analyticsTables.get(0).getColumns();
+    assertTrue(columns.stream().map(Column::getName).anyMatch("teaUid"::equals));
+    assertFalse(columns.stream().map(Column::getName).anyMatch("skippedTeaUid"::equals));
   }
 }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/notification/BaseNotificationMessageRenderer.java

@@ -72,8 +72,6 @@ public abstract class BaseNotificationMessageRenderer<T> implements Notification
 
   protected static final int SUBJECT_CHAR_LIMIT = 100;
 
-  protected static final String CONFIDENTIAL_VALUE_REPLACEMENT = "[CONFIDENTIAL]";
-
   // Error placeholders
   protected static final String MISSING_VALUE_REPLACEMENT = "[N/A]";
   protected static final String VALUE_ON_ERROR = "[SERVER ERROR]";
@@ -199,7 +197,6 @@ protected abstract Map<String, String> resolveDataElementValues(
    *
    * <ul>
    *   <li>Returns a placeholder if the {@code DataElement} is not part of the program stage.
-   *   <li>Returns a confidential replacement if the underlying value is {@code null}.
    *   <li>Resolves OptionSet codes to their corresponding display names when applicable.
    *   <li>Otherwise, returns the raw data value.
    * </ul>
@@ -214,10 +211,6 @@ protected String renderDataElementValue(EventDataValue dv, DataElement dataEleme
     }
     String value = dv.getValue();
 
-    if (value == null) {
-      return CONFIDENTIAL_VALUE_REPLACEMENT;
-    }
-
     // If the DV has an OptionSet -> substitute value with the name of the
     // Option
 

dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/trackedentity/hibernate/TrackedEntityAttribute.hbm.xml

@@ -62,8 +62,6 @@
 
     <property name="sortOrderInListNoProgram" />
 
-    <property name="confidential" />
-
     <property name="unique" column="uniquefield" />
 
     <property name="generated" />