Switch incompatible indices list to entity table

full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java

@@ -43,7 +43,7 @@ public class OutdatedIndexServiceIT extends SearchServerBaseTest {
     public void setUp() throws Exception {
         indicesAdapter = searchServer().adapters().indicesAdapter();
         countsAdapter = searchServer().adapters().countsAdapter();
-        outdatedIndexService = new OutdatedIndexService(indicesAdapter, null, null);
+        outdatedIndexService = new OutdatedIndexService(indicesAdapter, null, null, null, null);
     }
 
     @FullBackendTest

graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java

@@ -18,24 +18,99 @@
 
 package org.graylog2.indexer.indices;
 
-public record OutdatedIndex(String indexName, String version, boolean warmIndex,
-                            boolean managedIndex, String activeWriteIndex) implements Comparable<OutdatedIndex> {
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog2.indexer.ranges.IndexRange;
+import org.graylog2.utilities.lucene.InMemorySearchableEntity;
+import org.graylog2.utilities.lucene.LuceneDocBuilder;
+import org.joda.time.DateTime;
+
+public record OutdatedIndex(@JsonProperty(FIELD_INDEX_NAME) String indexName,
+                            @JsonProperty(FIELD_VERSION) String version,
+                            @JsonProperty(FIELD_WARM_INDEX) boolean warmIndex,
+                            @JsonProperty(FIELD_MANAGED_INDEX) boolean managedIndex,
+                            @JsonProperty(FIELD_ACTIVE_WRITE_INDEX) String activeWriteIndex,
+                            @JsonProperty(FIELD_BEGIN) DateTime begin,
+                            @JsonProperty(FIELD_END) DateTime end)
+        implements Comparable<OutdatedIndex>, InMemorySearchableEntity {
+
+    public static final String FIELD_INDEX_NAME = "index_name";
+    public static final String FIELD_VERSION = "version";
+    public static final String FIELD_WARM_INDEX = "warm_index";
+    public static final String FIELD_MANAGED_INDEX = "managed_index";
+    public static final String FIELD_ACTIVE_WRITE_INDEX = "active_write_index";
+    public static final String FIELD_SYSTEM_INDEX = "system_index";
+    public static final String FIELD_BEGIN = "begin";
+    public static final String FIELD_END = "end";
+    // Derived, multi-valued classification used for the single "type" filter column.
+    public static final String FIELD_CATEGORY = "category";
+    public static final String CATEGORY_GRAYLOG = "graylog";
+    public static final String CATEGORY_SYSTEM = "system";
+    public static final String CATEGORY_FOREIGN = "foreign";
+    public static final String CATEGORY_WARM = "warm";
+
     public OutdatedIndex(String indexName, String version, boolean warmIndex) {
-        this(indexName, version, warmIndex, false, null);
+        this(indexName, version, warmIndex, false, null, null, null);
+    }
+
+    public OutdatedIndex(String indexName, String version, boolean warmIndex, boolean managedIndex, String activeWriteIndex) {
+        this(indexName, version, warmIndex, managedIndex, activeWriteIndex, null, null);
     }
 
     public OutdatedIndex asManaged(boolean managed) {
-        return new OutdatedIndex(indexName, version, warmIndex, managed, activeWriteIndex);
+        return new OutdatedIndex(indexName, version, warmIndex, managed, activeWriteIndex, begin, end);
     }
 
     public OutdatedIndex asActiveWriteIndex(String isActiveWriteIndex) {
-        return new OutdatedIndex(indexName, version, warmIndex, managedIndex, isActiveWriteIndex);
+        return new OutdatedIndex(indexName, version, warmIndex, managedIndex, isActiveWriteIndex, begin, end);
+    }
+
+    public OutdatedIndex withRange(IndexRange range) {
+        if (range == null || isUnknownRange(range)) {
+            return new OutdatedIndex(indexName, version, warmIndex, managedIndex, activeWriteIndex, null, null);
+        }
+        return new OutdatedIndex(indexName, version, warmIndex, managedIndex, activeWriteIndex, range.begin(), range.end());
     }
 
+    private static boolean isUnknownRange(IndexRange range) {
+        return range.begin().getMillis() == 0L && range.end().getMillis() == 0L;
+    }
+
+    @JsonProperty(FIELD_SYSTEM_INDEX)
     public boolean isSystemIndex() {
         return indexName.startsWith(".");
     }
 
+    /**
+     * The mutually exclusive primary classification of this index. System indices take precedence over the
+     * managed/foreign distinction, mirroring the badges shown in the UI.
+     */
+    private String primaryCategory() {
+        if (isSystemIndex()) {
+            return CATEGORY_SYSTEM;
+        }
+        return managedIndex ? CATEGORY_GRAYLOG : CATEGORY_FOREIGN;
+    }
+
+    @JsonIgnore
+    @Override
+    public void buildLuceneDoc(LuceneDocBuilder builder) {
+        builder.stringVal(FIELD_INDEX_NAME, indexName);
+        builder.stringVal(FIELD_VERSION, version);
+        // category is multi-valued: the primary classification plus an optional "warm" token, matching the badges.
+        builder.searchableVal(FIELD_CATEGORY, primaryCategory());
+        if (warmIndex) {
+            builder.searchableVal(FIELD_CATEGORY, CATEGORY_WARM);
+        }
+        // dateVal is not null-safe, so only add the range fields when present.
+        if (begin != null) {
+            builder.dateVal(FIELD_BEGIN, begin.toDate());
+        }
+        if (end != null) {
+            builder.dateVal(FIELD_END, end.toDate());
+        }
+    }
+
     @Override
     public int compareTo(OutdatedIndex other) {
         return this.indexName.compareTo(other.indexName);

graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java

@@ -19,12 +19,16 @@
 
 import com.github.zafarkhaja.semver.ParseException;
 import com.github.zafarkhaja.semver.Version;
+import com.google.common.eventbus.EventBus;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 import jakarta.validation.constraints.NotNull;
 import org.graylog2.indexer.ElasticsearchException;
 import org.graylog2.indexer.cluster.Cluster;
 import org.graylog2.indexer.indexset.registry.IndexSetRegistry;
+import org.graylog2.indexer.indices.events.IndicesDeletedEvent;
+import org.graylog2.indexer.ranges.IndexRange;
+import org.graylog2.indexer.ranges.IndexRangeService;
 import org.graylog2.indexer.security.IndexerAdminCert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,6 +39,11 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static com.mongodb.client.model.Filters.in;
 
 @Singleton
 public class OutdatedIndexService {
@@ -43,12 +52,17 @@ public class OutdatedIndexService {
 
     private final IndicesAdapter indicesAdapter;
     private final IndexSetRegistry indexSetRegistry;
+    private final IndexRangeService indexRangeService;
+    private final EventBus eventBus;
     private final Cluster cluster;
 
     @Inject
-    public OutdatedIndexService(@IndexerAdminCert IndicesAdapter indicesAdapter, IndexSetRegistry indexSetRegistry, Cluster cluster) {
+    public OutdatedIndexService(@IndexerAdminCert IndicesAdapter indicesAdapter, IndexSetRegistry indexSetRegistry,
+                                IndexRangeService indexRangeService, EventBus eventBus, Cluster cluster) {
         this.indicesAdapter = indicesAdapter;
         this.indexSetRegistry = indexSetRegistry;
+        this.indexRangeService = indexRangeService;
+        this.eventBus = eventBus;
         this.cluster = cluster;
     }
 
@@ -61,12 +75,25 @@ public List<OutdatedIndex> getOutdatedIndices() {
                         throw new IllegalStateException("Cluster version cannot be determined: " + version);
                     }
                 }).orElseThrow(() -> new IllegalStateException("Cluster version cannot be determined: null"));
-        return indicesAdapter.getOutdatedIndices(currentMajorVersion).stream()
+        final List<OutdatedIndex> outdatedIndices = indicesAdapter.getOutdatedIndices(currentMajorVersion).stream()
                 .map(index -> index.asManaged(indexSetRegistry.isManagedIndex(index.indexName())))
                 .map(index -> index.asActiveWriteIndex(isActiveWriteIndexForSet(index.indexName())))
+                .toList();
+        final Map<String, IndexRange> ranges = findRanges(outdatedIndices);
+        return outdatedIndices.stream()
+                .map(index -> index.withRange(ranges.get(index.indexName())))
                 .sorted().toList();
     }
 
+    private Map<String, IndexRange> findRanges(List<OutdatedIndex> indices) {
+        final Set<String> indexNames = indices.stream().map(OutdatedIndex::indexName).collect(Collectors.toSet());
+        if (indexNames.isEmpty()) {
+            return Map.of();
+        }
+        return indexRangeService.find(in(IndexRange.FIELD_INDEX_NAME, indexNames)).stream()
+                .collect(Collectors.toMap(IndexRange::indexName, Function.identity(), (existing, replacement) -> existing));
+    }
+
     public String isActiveWriteIndexForSet(String index) {
         return indexSetRegistry.getForIndex(index)
                 .filter(indexSet -> Objects.equals(indexSet.getActiveWriteIndex(), index))
@@ -160,5 +187,7 @@ private Map<String, Object> cleanIndexSettings(Map<String, Object> settings, boo
 
     public void delete(@NotNull String index) {
         indicesAdapter.delete(index);
+        // Mirror Indices#delete so listeners clean up index ranges and cached field types for managed indices.
+        eventBus.post(IndicesDeletedEvent.create(index));
     }
 }

graylog2-server/src/main/java/org/graylog2/rest/resources/RestResourcesModule.java

@@ -97,6 +97,7 @@
 import org.graylog2.rest.resources.system.indexer.IndexerClusterResource;
 import org.graylog2.rest.resources.system.indexer.IndexerOverviewResource;
 import org.graylog2.rest.resources.system.indexer.IndicesResource;
+import org.graylog2.rest.resources.system.indexer.OutdatedIndexResource;
 import org.graylog2.rest.resources.system.indices.RetentionStrategyResource;
 import org.graylog2.rest.resources.system.indices.RotationStrategyResource;
 import org.graylog2.rest.resources.system.inputs.ExtractorsResource;
@@ -231,6 +232,7 @@ private void addIndexingResources() {
         addSystemRestResource(IndexSetDefaultsResource.class);
         addSystemRestResource(IndexTemplatesResource.class);
         addSystemRestResource(IndicesResource.class);
+        addSystemRestResource(OutdatedIndexResource.class);
         addSystemRestResource(IndexRangesResource.class);
         addSystemRestResource(RetentionStrategyResource.class);
         addSystemRestResource(RotationStrategyResource.class);

graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndicesResource.java

@@ -27,15 +27,13 @@
 import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotNull;
 import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.DefaultValue;
 import jakarta.ws.rs.ForbiddenException;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.NotFoundException;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.PathParam;
 import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.core.MediaType;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
 import org.apache.shiro.authz.annotation.RequiresPermissions;
@@ -47,7 +45,6 @@
 import org.graylog2.indexer.indexset.index.IndexPattern;
 import org.graylog2.indexer.indexset.registry.IndexSetRegistry;
 import org.graylog2.indexer.indices.Indices;
-import org.graylog2.indexer.indices.OutdatedIndex;
 import org.graylog2.indexer.indices.OutdatedIndexService;
 import org.graylog2.indexer.indices.TooManyAliasesException;
 import org.graylog2.indexer.indices.stats.IndexStatistics;
@@ -315,44 +312,6 @@ public ClosedIndices indexSetReopened(@Parameter(name = "indexSetId") @PathParam
         return ClosedIndices.create(reopenedIndices, reopenedIndices.size());
     }
 
-    @GET
-    @Path("/outdated")
-    @Operation(summary = "Get a list of indices that were created in a OpenSearch version prior to the recent one")
-    @RequiresPermissions(RestPermissions.INDICES_READ)
-    @Produces(MediaType.APPLICATION_JSON)
-    public List<OutdatedIndex> getOutdatedIndices() {
-        return outdatedIndexService.getOutdatedIndices();
-    }
-
-    @POST
-    @Path("/outdated/{index}/reindex")
-    @Operation(summary = "Reindexes an outdated index to make it compatible with the next major version of OpenSearch")
-    @RequiresPermissions(RestPermissions.INDICES_REINDEX)
-    @Produces(MediaType.APPLICATION_JSON)
-    @AuditEvent(type = AuditEventTypes.ES_INDEX_REINDEX)
-    public void reindex(@Parameter(name = "index") @PathParam("index") @NotNull String index,
-                        @Parameter(name = "withReplication") @QueryParam("withReplication") @DefaultValue("true") boolean withReplication) {
-        OutdatedIndex outdatedIndex = getOutdatedIndices().stream()
-                .filter(OutdatedIndex::isSystemIndex)
-                .filter(i -> i.indexName().equals(index))
-                .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is no system index"));
-        outdatedIndexService.reindex(outdatedIndex.indexName(), withReplication);
-    }
-
-    @DELETE
-    @Path("/outdated/{index}")
-    @Operation(summary = "Deletes an outdated, non-Graylog managed index")
-    @RequiresPermissions(RestPermissions.INDICES_DELETE)
-    @Produces(MediaType.APPLICATION_JSON)
-    @AuditEvent(type = AuditEventTypes.ES_INDEX_DELETE)
-    public void deleteOutdated(@Parameter(name = "index") @PathParam("index") @NotNull String index) {
-        OutdatedIndex outdatedIndex = getOutdatedIndices().stream()
-                .filter(i -> !i.managedIndex())
-                .filter(i -> i.indexName().equals(index))
-                .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is an index managed by Graylog"));
-        outdatedIndexService.delete(outdatedIndex.indexName());
-    }
-
     private OpenIndicesInfo getOpenIndicesInfo(Set<IndexStatistics> indicesStatistics) {
         final List<IndexInfo> indexInfos = new LinkedList<>();
         final Set<String> indices = indicesStatistics.stream()