From e9815f982c6f8de3de1a45068f2f14e11fd751b0 Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Wed, 15 Jul 2026 15:53:26 +0200 Subject: [PATCH 1/9] add entity table for outdated indices --- .../indices/OutdatedIndexServiceIT.java | 2 +- .../indexer/indices/OutdatedIndex.java | 67 ++++++++- .../indexer/indices/OutdatedIndexService.java | 27 +++- .../rest/resources/RestResourcesModule.java | 2 + .../system/indexer/OutdatedIndexResource.java | 136 ++++++++++++++++++ .../indices/OutdatedIndexServiceTest.java | 8 ++ 6 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java diff --git a/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java b/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java index 5a172e7f5639..a29ef64696ef 100644 --- a/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java +++ b/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java @@ -43,7 +43,7 @@ public class OutdatedIndexServiceIT extends SearchServerBaseTest { public void setUp() throws Exception { indicesAdapter = searchServer().adapters().indicesAdapter(); countsAdapter = searchServer().adapters().countsAdapter(); - outdatedIndexService = new OutdatedIndexService(indicesAdapter, null, null); + outdatedIndexService = new OutdatedIndexService(indicesAdapter, null, null, null); } @FullBackendTest diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java index 148d57cc5a70..c48bc07d7398 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java @@ -18,24 +18,81 @@ package org.graylog2.indexer.indices; -public record OutdatedIndex(String indexName, String version, boolean warmIndex, - boolean managedIndex, String activeWriteIndex) implements Comparable { +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog2.indexer.ranges.IndexRange; +import org.graylog2.utilities.lucene.InMemorySearchableEntity; +import org.graylog2.utilities.lucene.LuceneDocBuilder; +import org.joda.time.DateTime; + +public record OutdatedIndex(@JsonProperty(FIELD_INDEX_NAME) String indexName, + @JsonProperty(FIELD_VERSION) String version, + @JsonProperty(FIELD_WARM_INDEX) boolean warmIndex, + @JsonProperty(FIELD_MANAGED_INDEX) boolean managedIndex, + @JsonProperty(FIELD_ACTIVE_WRITE_INDEX) String activeWriteIndex, + @JsonProperty(FIELD_BEGIN) DateTime begin, + @JsonProperty(FIELD_END) DateTime end) + implements Comparable, InMemorySearchableEntity { + + public static final String FIELD_INDEX_NAME = "indexName"; + public static final String FIELD_VERSION = "version"; + public static final String FIELD_WARM_INDEX = "warmIndex"; + public static final String FIELD_MANAGED_INDEX = "managedIndex"; + public static final String FIELD_ACTIVE_WRITE_INDEX = "activeWriteIndex"; + public static final String FIELD_SYSTEM_INDEX = "systemIndex"; + public static final String FIELD_BEGIN = "begin"; + public static final String FIELD_END = "end"; + public OutdatedIndex(String indexName, String version, boolean warmIndex) { - this(indexName, version, warmIndex, false, null); + this(indexName, version, warmIndex, false, null, null, null); + } + + public OutdatedIndex(String indexName, String version, boolean warmIndex, boolean managedIndex, String activeWriteIndex) { + this(indexName, version, warmIndex, managedIndex, activeWriteIndex, null, null); } public OutdatedIndex asManaged(boolean managed) { - return new OutdatedIndex(indexName, version, warmIndex, managed, activeWriteIndex); + return new OutdatedIndex(indexName, version, warmIndex, managed, activeWriteIndex, begin, end); } public OutdatedIndex asActiveWriteIndex(String isActiveWriteIndex) { - return new OutdatedIndex(indexName, version, warmIndex, managedIndex, isActiveWriteIndex); + return new OutdatedIndex(indexName, version, warmIndex, managedIndex, isActiveWriteIndex, begin, end); + } + + public OutdatedIndex withRange(IndexRange range) { + if (range == null || isUnknownRange(range)) { + return new OutdatedIndex(indexName, version, warmIndex, managedIndex, activeWriteIndex, null, null); + } + return new OutdatedIndex(indexName, version, warmIndex, managedIndex, activeWriteIndex, range.begin(), range.end()); + } + + private static boolean isUnknownRange(IndexRange range) { + return range.begin().getMillis() == 0L && range.end().getMillis() == 0L; } + @JsonProperty(FIELD_SYSTEM_INDEX) public boolean isSystemIndex() { return indexName.startsWith("."); } + @JsonIgnore + @Override + public void buildLuceneDoc(LuceneDocBuilder builder) { + builder.stringVal(FIELD_INDEX_NAME, indexName); + builder.stringVal(FIELD_VERSION, version); + builder.boolVal(FIELD_WARM_INDEX, warmIndex); + builder.boolVal(FIELD_MANAGED_INDEX, managedIndex); + builder.stringVal(FIELD_ACTIVE_WRITE_INDEX, activeWriteIndex); + builder.boolVal(FIELD_SYSTEM_INDEX, isSystemIndex()); + // dateVal is not null-safe, so only add the range fields when present. + if (begin != null) { + builder.dateVal(FIELD_BEGIN, begin.toDate()); + } + if (end != null) { + builder.dateVal(FIELD_END, end.toDate()); + } + } + @Override public int compareTo(OutdatedIndex other) { return this.indexName.compareTo(other.indexName); diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java index ad165d3c7c27..92166fc85b29 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java @@ -25,6 +25,8 @@ import org.graylog2.indexer.ElasticsearchException; import org.graylog2.indexer.cluster.Cluster; import org.graylog2.indexer.indexset.registry.IndexSetRegistry; +import org.graylog2.indexer.ranges.IndexRange; +import org.graylog2.indexer.ranges.IndexRangeService; import org.graylog2.indexer.security.IndexerAdminCert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +37,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.mongodb.client.model.Filters.in; @Singleton public class OutdatedIndexService { @@ -43,12 +50,15 @@ public class OutdatedIndexService { private final IndicesAdapter indicesAdapter; private final IndexSetRegistry indexSetRegistry; + private final IndexRangeService indexRangeService; private final Cluster cluster; @Inject - public OutdatedIndexService(@IndexerAdminCert IndicesAdapter indicesAdapter, IndexSetRegistry indexSetRegistry, Cluster cluster) { + public OutdatedIndexService(@IndexerAdminCert IndicesAdapter indicesAdapter, IndexSetRegistry indexSetRegistry, + IndexRangeService indexRangeService, Cluster cluster) { this.indicesAdapter = indicesAdapter; this.indexSetRegistry = indexSetRegistry; + this.indexRangeService = indexRangeService; this.cluster = cluster; } @@ -61,12 +71,25 @@ public List getOutdatedIndices() { throw new IllegalStateException("Cluster version cannot be determined: " + version); } }).orElseThrow(() -> new IllegalStateException("Cluster version cannot be determined: null")); - return indicesAdapter.getOutdatedIndices(currentMajorVersion).stream() + final List outdatedIndices = indicesAdapter.getOutdatedIndices(currentMajorVersion).stream() .map(index -> index.asManaged(indexSetRegistry.isManagedIndex(index.indexName()))) .map(index -> index.asActiveWriteIndex(isActiveWriteIndexForSet(index.indexName()))) + .toList(); + final Map ranges = findRanges(outdatedIndices); + return outdatedIndices.stream() + .map(index -> index.withRange(ranges.get(index.indexName()))) .sorted().toList(); } + private Map findRanges(List indices) { + final Set indexNames = indices.stream().map(OutdatedIndex::indexName).collect(Collectors.toSet()); + if (indexNames.isEmpty()) { + return Map.of(); + } + return indexRangeService.find(in(IndexRange.FIELD_INDEX_NAME, indexNames)).stream() + .collect(Collectors.toMap(IndexRange::indexName, Function.identity(), (existing, replacement) -> existing)); + } + public String isActiveWriteIndexForSet(String index) { return indexSetRegistry.getForIndex(index) .filter(indexSet -> Objects.equals(indexSet.getActiveWriteIndex(), index)) diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/RestResourcesModule.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/RestResourcesModule.java index d0c18c737350..b9f27a6a68ef 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/RestResourcesModule.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/RestResourcesModule.java @@ -97,6 +97,7 @@ import org.graylog2.rest.resources.system.indexer.IndexerClusterResource; import org.graylog2.rest.resources.system.indexer.IndexerOverviewResource; import org.graylog2.rest.resources.system.indexer.IndicesResource; +import org.graylog2.rest.resources.system.indexer.OutdatedIndexResource; import org.graylog2.rest.resources.system.indices.RetentionStrategyResource; import org.graylog2.rest.resources.system.indices.RotationStrategyResource; import org.graylog2.rest.resources.system.inputs.ExtractorsResource; @@ -231,6 +232,7 @@ private void addIndexingResources() { addSystemRestResource(IndexSetDefaultsResource.class); addSystemRestResource(IndexTemplatesResource.class); addSystemRestResource(IndicesResource.class); + addSystemRestResource(OutdatedIndexResource.class); addSystemRestResource(IndexRangesResource.class); addSystemRestResource(RetentionStrategyResource.class); addSystemRestResource(RotationStrategyResource.class); diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java new file mode 100644 index 000000000000..111eafcbf7fe --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.rest.resources.system.indexer; + +import com.codahale.metrics.annotation.Timed; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.inject.Inject; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog2.database.PaginatedList; +import org.graylog2.indexer.indices.OutdatedIndex; +import org.graylog2.indexer.indices.OutdatedIndexService; +import org.graylog2.rest.models.SortOrder; +import org.graylog2.rest.models.tools.responses.PageListResponse; +import org.graylog2.rest.resources.entities.EntityAttribute; +import org.graylog2.rest.resources.entities.EntityDefaults; +import org.graylog2.rest.resources.entities.FilterOption; +import org.graylog2.rest.resources.entities.Sorting; +import org.graylog2.search.SearchQuery; +import org.graylog2.search.SearchQueryField; +import org.graylog2.search.SearchQueryParser; +import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.shared.security.RestPermissions; +import org.graylog2.utilities.lucene.InMemorySearchEngine; +import org.graylog2.utilities.lucene.LuceneInMemorySearchEngine; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@Tag(name = "System/Indexer/Indices", description = "Outdated index discovery") +@RequiresAuthentication +@Path("/system/indexer/outdated_indices") +@Produces(MediaType.APPLICATION_JSON) +public class OutdatedIndexResource extends RestResource { + + private static final String DEFAULT_SORT_FIELD = OutdatedIndex.FIELD_INDEX_NAME; + private static final String DEFAULT_SORT_DIRECTION = "asc"; + private static final List attributes = List.of( + EntityAttribute.builder().id(OutdatedIndex.FIELD_INDEX_NAME).title("Index Name").type(SearchQueryField.Type.STRING).sortable(true).searchable(true).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_VERSION).title("Version").type(SearchQueryField.Type.STRING).sortable(true).searchable(true).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_ACTIVE_WRITE_INDEX).title("Active Write Index").type(SearchQueryField.Type.STRING).sortable(true).searchable(true).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_WARM_INDEX).title("Warm Index").type(SearchQueryField.Type.BOOLEAN).sortable(true).filterable(true).filterOptions(booleanFilterOptions()).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_MANAGED_INDEX).title("Managed Index").type(SearchQueryField.Type.BOOLEAN).sortable(true).filterable(true).filterOptions(booleanFilterOptions()).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_SYSTEM_INDEX).title("System Index").type(SearchQueryField.Type.BOOLEAN).sortable(true).filterable(true).filterOptions(booleanFilterOptions()).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_BEGIN).title("Message Range Begin").type(SearchQueryField.Type.DATE).sortable(true).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_END).title("Message Range End").type(SearchQueryField.Type.DATE).sortable(true).build() + ); + + // Booleans are indexed as int 1/0 (see OutdatedIndex#buildLuceneDoc and LuceneQueryBuilder), so the filter + // values must be the numeric representations rather than "true"/"false". + private static Set booleanFilterOptions() { + return Set.of( + FilterOption.create("1", "True"), + FilterOption.create("0", "False") + ); + } + + private static final EntityDefaults settings = EntityDefaults.builder() + .sort(Sorting.create(DEFAULT_SORT_FIELD, Sorting.Direction.valueOf(DEFAULT_SORT_DIRECTION.toUpperCase(Locale.ROOT)))) + .build(); + + private final InMemorySearchEngine outdatedIndexSearchService; + private final SearchQueryParser searchQueryParser; + + @Inject + public OutdatedIndexResource(OutdatedIndexService outdatedIndexService) { + final Supplier> cachingSupplier = Suppliers.memoizeWithExpiration( + outdatedIndexService::getOutdatedIndices, + 10, + TimeUnit.SECONDS + ); + this.outdatedIndexSearchService = new LuceneInMemorySearchEngine<>(attributes, cachingSupplier); + this.searchQueryParser = new SearchQueryParser(DEFAULT_SORT_FIELD, attributes); + } + + @GET + @Timed + @RequiresPermissions(RestPermissions.INDICES_READ) + @Operation(summary = "Get a paginated list of indices that were created in an OpenSearch version prior to the recent one") + public PageListResponse listOutdatedIndices( + @Parameter(name = "page") @QueryParam("page") @DefaultValue("1") int page, + @Parameter(name = "per_page") @QueryParam("per_page") @DefaultValue("50") int perPage, + @Parameter(name = "query") @QueryParam("query") @DefaultValue("") String query, + @Parameter(name = "sort", + description = "The field to sort the result on", + required = true, + schema = @Schema(allowableValues = { + OutdatedIndex.FIELD_INDEX_NAME, + OutdatedIndex.FIELD_VERSION, + OutdatedIndex.FIELD_ACTIVE_WRITE_INDEX, + OutdatedIndex.FIELD_WARM_INDEX, + OutdatedIndex.FIELD_MANAGED_INDEX, + OutdatedIndex.FIELD_SYSTEM_INDEX, + OutdatedIndex.FIELD_BEGIN, + OutdatedIndex.FIELD_END + })) + @DefaultValue(DEFAULT_SORT_FIELD) @QueryParam("sort") String sort, + @Parameter(name = "order", description = "The sort direction", + schema = @Schema(allowableValues = {"asc", "desc"})) + @DefaultValue(DEFAULT_SORT_DIRECTION) @QueryParam("order") SortOrder order + ) throws QueryNodeException, IOException { + final SearchQuery parsedQuery = searchQueryParser.parse(query); + final PaginatedList result = outdatedIndexSearchService.search(parsedQuery, sort, order, page, perPage); + return PageListResponse.create(query, result.pagination(), + result.grandTotal().orElse(0L), sort, order, result.stream().toList(), attributes, settings); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java index 6adf74129d8c..75bd5fa84f4a 100644 --- a/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java +++ b/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java @@ -18,10 +18,13 @@ package org.graylog2.indexer.indices; import org.assertj.core.api.Assertions; +import org.bson.conversions.Bson; import org.graylog2.indexer.cluster.Cluster; import org.graylog2.indexer.indexset.IndexSet; import org.graylog2.indexer.indexset.IndexSetConfig; import org.graylog2.indexer.indexset.registry.IndexSetRegistry; +import org.graylog2.indexer.ranges.IndexRange; +import org.graylog2.indexer.ranges.IndexRangeService; import org.graylog2.system.stats.elasticsearch.ElasticsearchStats; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -32,6 +35,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,6 +63,9 @@ class OutdatedIndexServiceTest { @Mock IndexSetRegistry indexSetRegistry; + @Mock + IndexRangeService indexRangeService; + @InjectMocks OutdatedIndexService outdatedIndexService; @@ -95,6 +102,7 @@ void getOutdatedIndicesSucceeds() { when(noWriteIndex.getActiveWriteIndex()).thenReturn("another_index"); when(indexSetRegistry.getForIndex("outdated2")).thenReturn(Optional.of(noWriteIndex)); when(indicesAdapter.getOutdatedIndices(2)).thenReturn(outdatedIndices); + when(indexRangeService.find(any(Bson.class))).thenReturn(Collections.emptySortedSet()); assertThat(outdatedIndexService.getOutdatedIndices()).isEqualTo(List.of( new OutdatedIndex("outdated1", "1.3.0", false, true, "id1"), new OutdatedIndex("outdated2", "1.3.0", true, false, null) From a00beb4ba9ab79d879ba7e7897b292343cffc21c Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Wed, 15 Jul 2026 17:47:42 +0200 Subject: [PATCH 2/9] fix CamelCase --- .../org/graylog2/indexer/indices/OutdatedIndex.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java index c48bc07d7398..d50c36ef3633 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java @@ -34,12 +34,12 @@ public record OutdatedIndex(@JsonProperty(FIELD_INDEX_NAME) String indexName, @JsonProperty(FIELD_END) DateTime end) implements Comparable, InMemorySearchableEntity { - public static final String FIELD_INDEX_NAME = "indexName"; + public static final String FIELD_INDEX_NAME = "index_name"; public static final String FIELD_VERSION = "version"; - public static final String FIELD_WARM_INDEX = "warmIndex"; - public static final String FIELD_MANAGED_INDEX = "managedIndex"; - public static final String FIELD_ACTIVE_WRITE_INDEX = "activeWriteIndex"; - public static final String FIELD_SYSTEM_INDEX = "systemIndex"; + public static final String FIELD_WARM_INDEX = "warm_index"; + public static final String FIELD_MANAGED_INDEX = "managed_index"; + public static final String FIELD_ACTIVE_WRITE_INDEX = "active_write_index"; + public static final String FIELD_SYSTEM_INDEX = "system_index"; public static final String FIELD_BEGIN = "begin"; public static final String FIELD_END = "end"; From 61750635aa68b9b7eeb9ec0e2f1c6f7458deaa9b Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Wed, 15 Jul 2026 17:48:03 +0200 Subject: [PATCH 3/9] move outdated/incompatible functions to resource --- .../system/indexer/IndicesResource.java | 41 ---------------- .../system/indexer/OutdatedIndexResource.java | 49 ++++++++++++++++++- ....java => OutdatedIndicesResourceTest.java} | 12 ++--- 3 files changed, 54 insertions(+), 48 deletions(-) rename graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/{IndicesResourceTest.java => OutdatedIndicesResourceTest.java} (86%) diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndicesResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndicesResource.java index 6cb65c492caa..7b09c9bb2828 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndicesResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndicesResource.java @@ -27,7 +27,6 @@ import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.ForbiddenException; import jakarta.ws.rs.GET; import jakarta.ws.rs.NotFoundException; @@ -35,7 +34,6 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; -import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.MediaType; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; @@ -47,7 +45,6 @@ import org.graylog2.indexer.indexset.index.IndexPattern; import org.graylog2.indexer.indexset.registry.IndexSetRegistry; import org.graylog2.indexer.indices.Indices; -import org.graylog2.indexer.indices.OutdatedIndex; import org.graylog2.indexer.indices.OutdatedIndexService; import org.graylog2.indexer.indices.TooManyAliasesException; import org.graylog2.indexer.indices.stats.IndexStatistics; @@ -315,44 +312,6 @@ public ClosedIndices indexSetReopened(@Parameter(name = "indexSetId") @PathParam return ClosedIndices.create(reopenedIndices, reopenedIndices.size()); } - @GET - @Path("/outdated") - @Operation(summary = "Get a list of indices that were created in a OpenSearch version prior to the recent one") - @RequiresPermissions(RestPermissions.INDICES_READ) - @Produces(MediaType.APPLICATION_JSON) - public List getOutdatedIndices() { - return outdatedIndexService.getOutdatedIndices(); - } - - @POST - @Path("/outdated/{index}/reindex") - @Operation(summary = "Reindexes an outdated index to make it compatible with the next major version of OpenSearch") - @RequiresPermissions(RestPermissions.INDICES_REINDEX) - @Produces(MediaType.APPLICATION_JSON) - @AuditEvent(type = AuditEventTypes.ES_INDEX_REINDEX) - public void reindex(@Parameter(name = "index") @PathParam("index") @NotNull String index, - @Parameter(name = "withReplication") @QueryParam("withReplication") @DefaultValue("true") boolean withReplication) { - OutdatedIndex outdatedIndex = getOutdatedIndices().stream() - .filter(OutdatedIndex::isSystemIndex) - .filter(i -> i.indexName().equals(index)) - .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is no system index")); - outdatedIndexService.reindex(outdatedIndex.indexName(), withReplication); - } - - @DELETE - @Path("/outdated/{index}") - @Operation(summary = "Deletes an outdated, non-Graylog managed index") - @RequiresPermissions(RestPermissions.INDICES_DELETE) - @Produces(MediaType.APPLICATION_JSON) - @AuditEvent(type = AuditEventTypes.ES_INDEX_DELETE) - public void deleteOutdated(@Parameter(name = "index") @PathParam("index") @NotNull String index) { - OutdatedIndex outdatedIndex = getOutdatedIndices().stream() - .filter(i -> !i.managedIndex()) - .filter(i -> i.indexName().equals(index)) - .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is an index managed by Graylog")); - outdatedIndexService.delete(outdatedIndex.indexName()); - } - private OpenIndicesInfo getOpenIndicesInfo(Set indicesStatistics) { final List indexInfos = new LinkedList<>(); final Set indices = indicesStatistics.stream() diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java index 111eafcbf7fe..4cb0c91ef4c9 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java @@ -24,15 +24,22 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.inject.Inject; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.MediaType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog2.audit.AuditEventTypes; +import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.database.PaginatedList; import org.graylog2.indexer.indices.OutdatedIndex; import org.graylog2.indexer.indices.OutdatedIndexService; @@ -58,7 +65,7 @@ @Tag(name = "System/Indexer/Indices", description = "Outdated index discovery") @RequiresAuthentication -@Path("/system/indexer/outdated_indices") +@Path("/system/indexer/indices/outdated") @Produces(MediaType.APPLICATION_JSON) public class OutdatedIndexResource extends RestResource { @@ -88,11 +95,13 @@ private static Set booleanFilterOptions() { .sort(Sorting.create(DEFAULT_SORT_FIELD, Sorting.Direction.valueOf(DEFAULT_SORT_DIRECTION.toUpperCase(Locale.ROOT)))) .build(); + private final OutdatedIndexService outdatedIndexService; private final InMemorySearchEngine outdatedIndexSearchService; private final SearchQueryParser searchQueryParser; @Inject public OutdatedIndexResource(OutdatedIndexService outdatedIndexService) { + this.outdatedIndexService = outdatedIndexService; final Supplier> cachingSupplier = Suppliers.memoizeWithExpiration( outdatedIndexService::getOutdatedIndices, 10, @@ -102,10 +111,19 @@ public OutdatedIndexResource(OutdatedIndexService outdatedIndexService) { this.searchQueryParser = new SearchQueryParser(DEFAULT_SORT_FIELD, attributes); } + @GET + @Timed + @RequiresPermissions(RestPermissions.INDICES_READ) + @Operation(summary = "Get a list of indices that were created in an OpenSearch version prior to the recent one") + public List getOutdatedIndices() { + return outdatedIndexService.getOutdatedIndices(); + } + @GET @Timed @RequiresPermissions(RestPermissions.INDICES_READ) @Operation(summary = "Get a paginated list of indices that were created in an OpenSearch version prior to the recent one") + @Path("/paginated") public PageListResponse listOutdatedIndices( @Parameter(name = "page") @QueryParam("page") @DefaultValue("1") int page, @Parameter(name = "per_page") @QueryParam("per_page") @DefaultValue("50") int perPage, @@ -133,4 +151,33 @@ public PageListResponse listOutdatedIndices( return PageListResponse.create(query, result.pagination(), result.grandTotal().orElse(0L), sort, order, result.stream().toList(), attributes, settings); } + + @POST + @Path("/{index}/reindex") + @Operation(summary = "Reindexes an outdated index to make it compatible with the next major version of OpenSearch") + @RequiresPermissions(RestPermissions.INDICES_REINDEX) + @Produces(MediaType.APPLICATION_JSON) + @AuditEvent(type = AuditEventTypes.ES_INDEX_REINDEX) + public void reindex(@Parameter(name = "index") @PathParam("index") @NotNull String index, + @Parameter(name = "withReplication") @QueryParam("withReplication") @DefaultValue("true") boolean withReplication) { + OutdatedIndex outdatedIndex = getOutdatedIndices().stream() + .filter(OutdatedIndex::isSystemIndex) + .filter(i -> i.indexName().equals(index)) + .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is no system index")); + outdatedIndexService.reindex(outdatedIndex.indexName(), withReplication); + } + + @DELETE + @Path("/{index}") + @Operation(summary = "Deletes an outdated, non-Graylog managed index") + @RequiresPermissions(RestPermissions.INDICES_DELETE) + @Produces(MediaType.APPLICATION_JSON) + @AuditEvent(type = AuditEventTypes.ES_INDEX_DELETE) + public void deleteOutdated(@Parameter(name = "index") @PathParam("index") @NotNull String index) { + OutdatedIndex outdatedIndex = getOutdatedIndices().stream() + .filter(i -> !i.managedIndex()) + .filter(i -> i.indexName().equals(index)) + .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is an index managed by Graylog")); + outdatedIndexService.delete(outdatedIndex.indexName()); + } } diff --git a/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/IndicesResourceTest.java b/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java similarity index 86% rename from graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/IndicesResourceTest.java rename to graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java index 7fa508f13164..9ec443423248 100644 --- a/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/IndicesResourceTest.java +++ b/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java @@ -41,7 +41,7 @@ @ExtendWith(MockitoExtension.class) @ExtendWith(WithAuthorizationExtension.class) -class IndicesResourceTest { +class OutdatedIndicesResourceTest { @Mock Indices indices; @@ -56,12 +56,12 @@ class IndicesResourceTest { OutdatedIndexService outdatedIndexService; @InjectMocks - IndicesResource indicesResource; + OutdatedIndexResource outdatedIndexResource; @Test @WithAuthorization(permissions = {"something:else"}) void getOutdatedIndicesFailsIfNotPermitted() { - Assertions.assertThatThrownBy(() -> indicesResource.getOutdatedIndices()).isInstanceOf(ForbiddenException.class); + Assertions.assertThatThrownBy(() -> outdatedIndexResource.getOutdatedIndices()).isInstanceOf(ForbiddenException.class); } @Test @@ -72,13 +72,13 @@ void getOutdatedIndicesSucceeds() { new OutdatedIndex("outdated2", "1.3.0", true, true, "id1") ); when(outdatedIndexService.getOutdatedIndices()).thenReturn(outdatedIndices); - assertThat(indicesResource.getOutdatedIndices()).isEqualTo(outdatedIndices); + assertThat(outdatedIndexResource.getOutdatedIndices()).isEqualTo(outdatedIndices); } @Test @WithAuthorization(permissions = {"something:else"}) void reindexFailsIfNotPermitted() { - Assertions.assertThatThrownBy(() -> indicesResource.reindex("outdated", false)) + Assertions.assertThatThrownBy(() -> outdatedIndexResource.reindex("outdated", false)) .isInstanceOf(ForbiddenException.class); } @@ -86,7 +86,7 @@ void reindexFailsIfNotPermitted() { @WithAuthorization(permissions = {"indices:read", "indices:reindex"}) void reindexSucceeds() { when(outdatedIndexService.getOutdatedIndices()).thenReturn(List.of(new OutdatedIndex(".outdated1", "1.3.0", false, false, null))); - indicesResource.reindex(".outdated1", true); + outdatedIndexResource.reindex(".outdated1", true); verify(outdatedIndexService, times(1)).reindex(".outdated1", true); } From ffe66619a991922af7f12f04e3064b360b4c40b4 Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Thu, 16 Jul 2026 07:33:11 +0200 Subject: [PATCH 4/9] add category and filter --- .../indexer/indices/OutdatedIndex.java | 26 ++++++++++++++++--- .../system/indexer/OutdatedIndexResource.java | 19 +++++--------- .../utilities/lucene/LuceneDocBuilder.java | 11 ++++++++ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java index d50c36ef3633..38e92a061224 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndex.java @@ -42,6 +42,12 @@ public record OutdatedIndex(@JsonProperty(FIELD_INDEX_NAME) String indexName, public static final String FIELD_SYSTEM_INDEX = "system_index"; public static final String FIELD_BEGIN = "begin"; public static final String FIELD_END = "end"; + // Derived, multi-valued classification used for the single "type" filter column. + public static final String FIELD_CATEGORY = "category"; + public static final String CATEGORY_GRAYLOG = "graylog"; + public static final String CATEGORY_SYSTEM = "system"; + public static final String CATEGORY_FOREIGN = "foreign"; + public static final String CATEGORY_WARM = "warm"; public OutdatedIndex(String indexName, String version, boolean warmIndex) { this(indexName, version, warmIndex, false, null, null, null); @@ -75,15 +81,27 @@ public boolean isSystemIndex() { return indexName.startsWith("."); } + /** + * The mutually exclusive primary classification of this index. System indices take precedence over the + * managed/foreign distinction, mirroring the badges shown in the UI. + */ + private String primaryCategory() { + if (isSystemIndex()) { + return CATEGORY_SYSTEM; + } + return managedIndex ? CATEGORY_GRAYLOG : CATEGORY_FOREIGN; + } + @JsonIgnore @Override public void buildLuceneDoc(LuceneDocBuilder builder) { builder.stringVal(FIELD_INDEX_NAME, indexName); builder.stringVal(FIELD_VERSION, version); - builder.boolVal(FIELD_WARM_INDEX, warmIndex); - builder.boolVal(FIELD_MANAGED_INDEX, managedIndex); - builder.stringVal(FIELD_ACTIVE_WRITE_INDEX, activeWriteIndex); - builder.boolVal(FIELD_SYSTEM_INDEX, isSystemIndex()); + // category is multi-valued: the primary classification plus an optional "warm" token, matching the badges. + builder.searchableVal(FIELD_CATEGORY, primaryCategory()); + if (warmIndex) { + builder.searchableVal(FIELD_CATEGORY, CATEGORY_WARM); + } // dateVal is not null-safe, so only add the range fields when present. if (begin != null) { builder.dateVal(FIELD_BEGIN, begin.toDate()); diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java index 4cb0c91ef4c9..e57a32e45e4e 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java @@ -73,21 +73,18 @@ public class OutdatedIndexResource extends RestResource { private static final String DEFAULT_SORT_DIRECTION = "asc"; private static final List attributes = List.of( EntityAttribute.builder().id(OutdatedIndex.FIELD_INDEX_NAME).title("Index Name").type(SearchQueryField.Type.STRING).sortable(true).searchable(true).build(), + EntityAttribute.builder().id(OutdatedIndex.FIELD_CATEGORY).title("Type").type(SearchQueryField.Type.STRING).sortable(false).filterable(true).filterOptions(categoryFilterOptions()).build(), EntityAttribute.builder().id(OutdatedIndex.FIELD_VERSION).title("Version").type(SearchQueryField.Type.STRING).sortable(true).searchable(true).build(), - EntityAttribute.builder().id(OutdatedIndex.FIELD_ACTIVE_WRITE_INDEX).title("Active Write Index").type(SearchQueryField.Type.STRING).sortable(true).searchable(true).build(), - EntityAttribute.builder().id(OutdatedIndex.FIELD_WARM_INDEX).title("Warm Index").type(SearchQueryField.Type.BOOLEAN).sortable(true).filterable(true).filterOptions(booleanFilterOptions()).build(), - EntityAttribute.builder().id(OutdatedIndex.FIELD_MANAGED_INDEX).title("Managed Index").type(SearchQueryField.Type.BOOLEAN).sortable(true).filterable(true).filterOptions(booleanFilterOptions()).build(), - EntityAttribute.builder().id(OutdatedIndex.FIELD_SYSTEM_INDEX).title("System Index").type(SearchQueryField.Type.BOOLEAN).sortable(true).filterable(true).filterOptions(booleanFilterOptions()).build(), EntityAttribute.builder().id(OutdatedIndex.FIELD_BEGIN).title("Message Range Begin").type(SearchQueryField.Type.DATE).sortable(true).build(), EntityAttribute.builder().id(OutdatedIndex.FIELD_END).title("Message Range End").type(SearchQueryField.Type.DATE).sortable(true).build() ); - // Booleans are indexed as int 1/0 (see OutdatedIndex#buildLuceneDoc and LuceneQueryBuilder), so the filter - // values must be the numeric representations rather than "true"/"false". - private static Set booleanFilterOptions() { + private static Set categoryFilterOptions() { return Set.of( - FilterOption.create("1", "True"), - FilterOption.create("0", "False") + FilterOption.create(OutdatedIndex.CATEGORY_GRAYLOG, "Graylog"), + FilterOption.create(OutdatedIndex.CATEGORY_SYSTEM, "System"), + FilterOption.create(OutdatedIndex.CATEGORY_FOREIGN, "Foreign"), + FilterOption.create(OutdatedIndex.CATEGORY_WARM, "Warm") ); } @@ -134,10 +131,6 @@ public PageListResponse listOutdatedIndices( schema = @Schema(allowableValues = { OutdatedIndex.FIELD_INDEX_NAME, OutdatedIndex.FIELD_VERSION, - OutdatedIndex.FIELD_ACTIVE_WRITE_INDEX, - OutdatedIndex.FIELD_WARM_INDEX, - OutdatedIndex.FIELD_MANAGED_INDEX, - OutdatedIndex.FIELD_SYSTEM_INDEX, OutdatedIndex.FIELD_BEGIN, OutdatedIndex.FIELD_END })) diff --git a/graylog2-server/src/main/java/org/graylog2/utilities/lucene/LuceneDocBuilder.java b/graylog2-server/src/main/java/org/graylog2/utilities/lucene/LuceneDocBuilder.java index 1d8f51fe4d2f..de66a85c7d76 100644 --- a/graylog2-server/src/main/java/org/graylog2/utilities/lucene/LuceneDocBuilder.java +++ b/graylog2-server/src/main/java/org/graylog2/utilities/lucene/LuceneDocBuilder.java @@ -41,6 +41,17 @@ public LuceneDocBuilder stringVal(String key, String value) { return this; } + /** + * Adds a searchable (but not sortable) string value. Unlike {@link #stringVal(String, String)} this does not add a + * {@link SortedDocValuesField}, so it may be called multiple times for the same key to index a multi-valued field. + */ + public LuceneDocBuilder searchableVal(String key, String value) { + if (value != null) { + doc.add(new TextField(key, value, Field.Store.NO)); + } + return this; + } + public LuceneDocBuilder dateVal(String key, Date value) { return longVal(key, value.getTime()); } From 076104b7e585a334826e7d9916317d5ec3851134 Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Thu, 16 Jul 2026 08:08:11 +0200 Subject: [PATCH 5/9] switch frontend to entity table --- .../IncompatibleIndexTableActions.tsx | 172 ++++++ .../IncompatibleIndicesBulkActions.tsx | 125 +++++ .../IncompatibleIndicesColumnRenderers.tsx | 99 ++++ .../IncompatibleIndicesTable.test.tsx | 509 +++--------------- .../IncompatibleIndicesTable.tsx | 308 +++-------- .../opensearch-upgrade/IndicesGroupTable.tsx | 252 --------- .../fetchIncompatibleIndices.ts | 57 ++ .../incompatibleIndexActions.tsx | 6 +- .../incompatibleIndexGroups.test.ts | 85 --- .../incompatibleIndexGroups.ts | 67 --- .../datanode/opensearch-upgrade/telemetry.ts | 17 + .../indices/hooks/useIncompatibleIndices.ts | 8 +- 12 files changed, 630 insertions(+), 1075 deletions(-) create mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx create mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx create mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx delete mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/IndicesGroupTable.tsx create mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts delete mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.test.ts delete mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.ts create mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/telemetry.ts diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx new file mode 100644 index 000000000000..333fe9240b3c --- /dev/null +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import React, { useState } from 'react'; +import styled from 'styled-components'; + +import { Button, ButtonToolbar, Label } from 'components/bootstrap'; +import { ConfirmDialog, ProgressBar } from 'components/common'; +import useSelectedEntities from 'components/common/EntityDataTable/hooks/useSelectedEntities'; +import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; +import extractErrorMessage from 'util/extractErrorMessage'; +import UserNotification from 'util/UserNotification'; + +import { TELEMETRY_DEFAULTS } from './telemetry'; +import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; +import type { PendingIndexStatus } from './hooks/usePendingIncompatibleIndexActions'; +import { getAvailableActions, useIncompatibleIndexActionDefinitions } from './incompatibleIndexActions'; +import type { IndexAction, PendingArchiveTracking } from './incompatibleIndexActions'; + +const ActionsToolbar = styled(ButtonToolbar)` + justify-content: flex-end; +`; + +const ArchiveProgressBar = styled(ProgressBar)` + display: inline-flex; + width: 120px; + margin-bottom: 0; + vertical-align: middle; +`; + +type Props = { + index: IncompatibleIndexRow; + canArchive: boolean; + pendingStatus: PendingIndexStatus | undefined; + isArchived: boolean; + addArchiveDeleteAction: (tracking: PendingArchiveTracking) => void; + refetchClusterJobs?: () => void; + refetch: () => void; +}; + +const IncompatibleIndexTableActions = ({ + index, + canArchive, + pendingStatus, + isArchived, + addArchiveDeleteAction, + refetchClusterJobs, + refetch, +}: Props) => { + const actionDefinitions = useIncompatibleIndexActionDefinitions(); + const sendTelemetry = useSendTelemetry(); + const { deselectEntity } = useSelectedEntities(); + const [confirmedAction, setConfirmedAction] = useState(); + const [isSubmitting, setIsSubmitting] = useState(false); + + if (pendingStatus?.state === 'archiving') { + // No empty 0% bar flash for jobs that finish almost instantly. + return pendingStatus.percent > 0 ? ( + + ) : ( + + ); + } + + const actions = getAvailableActions(index, canArchive, isArchived); + + const handleConfirm = async () => { + if (!confirmedAction) { + return; + } + + const actionDefinition = actionDefinitions[confirmedAction]; + sendTelemetry(actionDefinition.telemetryEventType, { ...TELEMETRY_DEFAULTS }); + setIsSubmitting(true); + + try { + const actionResponse = await actionDefinition.run(index); + const pendingArchive = actionDefinition.getPendingArchiveTracking?.(index, actionResponse); + + if (pendingArchive) { + addArchiveDeleteAction(pendingArchive); + } + + UserNotification.success(actionDefinition.successMessage(index)); + + if (confirmedAction === 'delete') { + deselectEntity(index.id); + } + + refetch(); + setConfirmedAction(undefined); + } catch (errorThrown) { + const errorMessage = extractErrorMessage(errorThrown); + + if (actionDefinition.isArchiveJobConflict?.(errorMessage)) { + UserNotification.warning( + 'Another archive job is already running. New archive jobs can be started after it finishes.', + 'Archive job already running', + ); + refetchClusterJobs?.(); + setConfirmedAction(undefined); + } else { + UserNotification.error(errorMessage, `Could not ${actionDefinition.confirmText.toLowerCase()}.`); + } + } finally { + setIsSubmitting(false); + } + }; + + const confirmDefinition = confirmedAction ? actionDefinitions[confirmedAction] : undefined; + + return ( + + {pendingStatus?.state === 'failed' && ( + + )} + {actions.map((action) => { + const actionDefinition = actionDefinitions[action]; + + return ( + + ); + })} + {confirmDefinition && ( + setConfirmedAction(undefined)} + onConfirm={handleConfirm} + submitLoadingText="Working..."> + {confirmDefinition.confirmationBody(index)} + + )} + + ); +}; + +export default IncompatibleIndexTableActions; diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx new file mode 100644 index 000000000000..081642fcd6d7 --- /dev/null +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import React, { useMemo, useState } from 'react'; + +import { MenuItem } from 'components/bootstrap'; +import BulkActionsDropdown from 'components/common/EntityDataTable/BulkActionsDropdown'; +import useSelectedEntities from 'components/common/EntityDataTable/hooks/useSelectedEntities'; +import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; +import UserNotification from 'util/UserNotification'; + +import { TELEMETRY_DEFAULTS } from './telemetry'; +import BulkIndexActionConfirmDialog from './BulkIndexActionConfirmDialog'; +import { CORE_ACTION_DEFINITIONS } from './incompatibleIndexActions'; +import { + getBulkIndexActionCandidates, + getBulkIndexActionNotification, + runBulkIndexAction, +} from './bulkIndexActions'; +import type { BulkIndexActionCandidate } from './bulkIndexActions'; +import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; +import type { PendingIndexStatus } from './hooks/usePendingIncompatibleIndexActions'; + +type Props = { + indices: Array; + canArchive: boolean; + pendingIndexStatuses: Map; + archivedIndexNames: ReadonlySet; + refetch: () => void; +}; + +const IncompatibleIndicesBulkActions = ({ + indices, + canArchive, + pendingIndexStatuses, + archivedIndexNames, + refetch, +}: Props) => { + const sendTelemetry = useSendTelemetry(); + const { selectedEntities, setSelectedEntities } = useSelectedEntities(); + const [confirmedBulkAction, setConfirmedBulkAction] = useState(); + const [isSubmitting, setIsSubmitting] = useState(false); + + const selectedIndices = useMemo( + () => indices.filter((index) => selectedEntities.includes(index.id)), + [indices, selectedEntities], + ); + + const candidates = useMemo( + () => getBulkIndexActionCandidates({ indices: selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames }), + [selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames], + ); + + const handleConfirm = async () => { + if (!confirmedBulkAction || isSubmitting) { + return; + } + + sendTelemetry(CORE_ACTION_DEFINITIONS[confirmedBulkAction.action].telemetryEventType, { + ...TELEMETRY_DEFAULTS, + app_action_value: 'bulk', + bulk_count: confirmedBulkAction.targetIndices.length, + }); + setIsSubmitting(true); + + try { + const result = await runBulkIndexAction({ + action: confirmedBulkAction.action, + indices: confirmedBulkAction.targetIndices, + }); + + const notification = getBulkIndexActionNotification(confirmedBulkAction, result); + + if (notification.type === 'success') { + UserNotification.success(notification.message); + } else if (notification.type === 'warning') { + UserNotification.warning(notification.message, notification.title); + } else { + UserNotification.error(notification.message, notification.title); + } + + const succeededIds = new Set(result.successes.map(({ index }) => index.index_name)); + setSelectedEntities(selectedEntities.filter((id) => !succeededIds.has(id))); + refetch(); + setConfirmedBulkAction(undefined); + } finally { + setIsSubmitting(false); + } + }; + + return ( + <> + + {candidates.map((candidate) => ( + setConfirmedBulkAction(candidate)}> + {candidate.buttonLabel} ({candidate.targetIndices.length}) + + ))} + + {confirmedBulkAction && ( + setConfirmedBulkAction(undefined)} + onConfirm={handleConfirm} + /> + )} + + ); +}; + +export default IncompatibleIndicesBulkActions; diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx new file mode 100644 index 000000000000..65737575b4a4 --- /dev/null +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; + +import type { ColorVariant } from '@graylog/sawmill'; + +import { Label } from 'components/bootstrap'; +import { Timestamp } from 'components/common'; +import type { ColumnRenderers } from 'components/common/EntityDataTable'; +import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; + +import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; +import type { PendingIndexStatus } from './hooks/usePendingIncompatibleIndexActions'; + +export const DEFAULT_DISPLAYED_COLUMNS = ['index_name', 'category', 'version', 'begin', 'end']; + +type Badge = { text: string; style: ColorVariant }; + +// The primary, mutually exclusive classification; matches the backend `category` field precedence. +const primaryTypeBadge = (index: IncompatibleIndex): Badge => { + if (index.system_index) { + return { text: 'System', style: 'info' }; + } + + return index.managed_index ? { text: 'Graylog', style: 'success' } : { text: 'Foreign', style: 'warning' }; +}; + +const typeBadges = (index: IncompatibleIndex): Array => [ + primaryTypeBadge(index), + ...(index.warm_index ? [{ text: 'Warm', style: 'default' as const }] : []), +]; + +const isIndexArchived = ( + indexName: string, + pendingStatus: PendingIndexStatus | undefined, + archivedIndexNames: ReadonlySet, +) => pendingStatus?.state !== 'archiving' && (archivedIndexNames.has(indexName) || pendingStatus?.state === 'archived'); + +const statusBadges = (index: IncompatibleIndex, isArchived: boolean): Array => [ + ...(index.active_write_index ? [{ text: 'active write index', style: 'default' as const }] : []), + ...(isArchived ? [{ text: 'archived already', style: 'success' as const }] : []), +]; + +const renderBadges = (badges: Array) => + badges.map(({ text, style }) => ( + + +   + + )); + +const renderRange = (value: unknown) => (value ? : —); + +export const createColumnRenderers = ( + pendingIndexStatuses: Map, + archivedIndexNames: ReadonlySet, +): ColumnRenderers => ({ + attributes: { + index_name: { + minWidth: 300, + renderCell: (_value, index) => { + const isArchived = isIndexArchived(index.index_name, pendingIndexStatuses.get(index.index_name), archivedIndexNames); + + return ( + + {index.index_name} +   + {renderBadges(statusBadges(index, isArchived))} + + ); + }, + }, + category: { + staticWidth: 200, + renderCell: (_value, index) => {renderBadges(typeBadges(index))}, + }, + version: { + renderCell: (_value, index) => index.version || 'Unknown', + }, + begin: { renderCell: renderRange }, + end: { renderCell: renderRange }, + }, +}); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx index 6c257768f1b4..04657d66f5a7 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx @@ -15,498 +15,121 @@ * . */ import * as React from 'react'; -import { render, screen, waitFor, within } from 'wrappedTestingLibrary'; -import userEvent from '@testing-library/user-event'; - -import { ClusterDeflector, IndexerIndices } from '@graylog/server-api'; +import { render, screen } from 'wrappedTestingLibrary'; import asMock from 'helpers/mocking/AsMock'; -import type { IndexArchiveBinding } from 'components/indices/archive/types'; -import useIndexArchive from 'components/indices/archive/useIndexArchive'; -import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; -import useIncompatibleIndices from 'components/indices/hooks/useIncompatibleIndices'; +import type { PaginatedEntityTableProps } from 'components/common/PaginatedEntityTable/PaginatedEntityTable'; import useCanArchive from 'components/indices/hooks/useCanArchive'; -import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; -import { TELEMETRY_EVENT_TYPE } from 'logic/telemetry/Constants'; -import UserNotification from 'util/UserNotification'; import IncompatibleIndicesTable from './IncompatibleIndicesTable'; -import { PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY } from './hooks/usePendingIncompatibleIndexActions'; -import useClusterJobs from './hooks/useClusterJobs'; -import type { SystemJobSummary } from './hooks/useClusterJobs'; +import { createColumnRenderers } from './IncompatibleIndicesColumnRenderers'; +import { fetchIncompatibleIndices, incompatibleIndicesKeyFn } from './fetchIncompatibleIndices'; +import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; import useArchivedIndexNames from './hooks/useArchivedIndexNames'; +import usePendingIncompatibleIndexActions from './hooks/usePendingIncompatibleIndexActions'; + +jest.mock('components/common/PaginatedEntityTable', () => ({ + __esModule: true, + default: jest.fn(({ humanName }) =>
Paginated {humanName}
), + useTableFetchContext: jest.fn(), +})); -jest.mock('components/indices/archive/useIndexArchive'); -jest.mock('components/indices/hooks/useIncompatibleIndices'); jest.mock('components/indices/hooks/useCanArchive'); -jest.mock('logic/telemetry/useSendTelemetry'); -jest.mock('./hooks/useClusterJobs'); jest.mock('./hooks/useArchivedIndexNames'); -jest.mock('logic/rest/FetchProvider', () => { - class Builder { - setHeader() { - return this; - } - - setHeaders() { - return this; - } - - json() { - return this; - } - - text() { - return this; - } - - raw() { - return this; - } - - build() { - return Promise.resolve(this); - } - } +jest.mock('./hooks/usePendingIncompatibleIndexActions'); - return { - __esModule: true, - default: jest.fn(() => Promise.resolve({ system_job: { id: 'archive-job-id' } })), - Builder, - }; -}); -jest.mock('@graylog/server-api', () => ({ - IndexerIndices: { - deleteOutdated: jest.fn(() => Promise.resolve()), - remove: jest.fn(() => Promise.resolve()), - reindex: jest.fn(() => Promise.resolve()), - }, - ClusterDeflector: { - cycleByindexSetId: jest.fn(() => Promise.resolve()), - }, -})); -jest.mock('util/UserNotification', () => ({ success: jest.fn(), warning: jest.fn(), error: jest.fn() })); - -const makeIndex = (overrides: Partial): IncompatibleIndex => ({ +const makeIndex = (overrides: Partial): IncompatibleIndexRow => ({ + id: 'index', index_name: 'index', version: '7.10.2', warm_index: false, managed_index: false, system_index: false, active_write_index: null, + begin: null, + end: null, ...overrides, }); -const graylogIndex = makeIndex({ index_name: 'graylog_0', managed_index: true }); -const secondGraylogIndex = makeIndex({ index_name: 'graylog_1', managed_index: true }); -const systemIndex = makeIndex({ index_name: '.system-index', system_index: true }); -const foreignIndex = makeIndex({ index_name: 'legacy-index' }); -const writeIndex = makeIndex({ index_name: 'graylog_2', managed_index: true, active_write_index: 'index-set-id' }); - -const mockIncompatibleIndices = (overrides: Partial>) => { - asMock(useIncompatibleIndices).mockReturnValue({ - data: [], - isError: false, - isLoading: false, - refetch: jest.fn(() => Promise.resolve({ data: [] })), - ...overrides, - } as ReturnType); -}; - -const ACTION_STARTED_AT = '2026-07-02T08:00:00.000Z'; -const JOBS_POLLED_AFTER_ACTION = Date.parse('2026-07-02T09:00:00.000Z'); -const JOBS_POLLED_BEFORE_ACTION = Date.parse('2026-07-02T07:00:00.000Z'); - -const clusterJob = (overrides: Partial): SystemJobSummary => - ({ - id: 'job-1', - name: 'archive-job', - description: 'Archiving index', - info: '', - job_status: 'running', - percent_complete: 0, - provides_progress: true, - is_cancelable: true, - execution_duration: 'PT1S', - started_at: ACTION_STARTED_AT, - node_id: 'node-1', - ...overrides, - }) as SystemJobSummary; - -const archiveBinding = (): IndexArchiveBinding => ({ - useCanArchive: () => true, - useArchivedIndexNames: () => new Set(), - archiveAndDeleteIndex: jest.fn(() => Promise.resolve({ systemJobId: 'archive-job-id' })), - isArchiveJobConflict: (errorMessage) => errorMessage.includes('already running'), - archiveSystemJobName: 'org.graylog.plugins.archive.job.ArchiveCreateSystemJob', -}); - -const storePendingArchive = (indexName: string, systemJobId?: string) => { - window.localStorage.setItem( - PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY, - JSON.stringify([{ action: 'archive-delete', indexName, systemJobId, startedAt: ACTION_STARTED_AT }]), - ); -}; - describe('IncompatibleIndicesTable', () => { beforeEach(() => { jest.clearAllMocks(); - window.localStorage.removeItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY); asMock(useCanArchive).mockReturnValue(true); - asMock(useIndexArchive).mockReturnValue(archiveBinding()); - asMock(useSendTelemetry).mockReturnValue(jest.fn()); - asMock(useClusterJobs).mockReturnValue({ jobsById: new Map(), jobsUpdatedAt: 0 }); asMock(useArchivedIndexNames).mockReturnValue(new Set()); - mockIncompatibleIndices({}); - }); - - it('shows a spinner while loading', async () => { - mockIncompatibleIndices({ isLoading: true }); - render(); - - expect(await screen.findByText(/loading incompatible indices/i)).toBeInTheDocument(); - }); - - it('shows an error alert when loading fails', () => { - mockIncompatibleIndices({ isError: true }); - render(); - - expect(screen.getByText(/could not load incompatible indices/i)).toBeInTheDocument(); - }); - - it('retries loading incompatible indices on demand from the error state', async () => { - const refetch = jest.fn(() => Promise.resolve({ data: [] })) as unknown as ReturnType< - typeof useIncompatibleIndices - >['refetch']; - mockIncompatibleIndices({ isError: true, refetch }); - render(); - - await userEvent.click(screen.getByRole('button', { name: /retry now/i })); - - expect(refetch).toHaveBeenCalled(); + asMock(usePendingIncompatibleIndexActions).mockReturnValue({ + pendingIndexStatuses: new Map(), + addArchiveDeleteAction: jest.fn(), + isArchiveJobRunning: false, + refetchClusterJobs: jest.fn(), + }); }); - it('shows a success message when there are no incompatible indices', () => { - mockIncompatibleIndices({ data: [] }); - render(); - - expect(screen.getByText(/no incompatible indices found/i)).toBeInTheDocument(); - }); + it('wires the paginated entity table to the outdated indices endpoint', () => { + const { default: PaginatedEntityTable } = jest.requireMock('components/common/PaginatedEntityTable'); + const mockPaginatedEntityTable = asMock(PaginatedEntityTable); - it('renders the group counts and the default group rows', () => { - mockIncompatibleIndices({ data: [graylogIndex, systemIndex] }); render(); - expect(screen.getByText('Graylog (1)')).toBeInTheDocument(); - expect(screen.getByText('System (1)')).toBeInTheDocument(); - expect(screen.getByText('Foreign (0)')).toBeInTheDocument(); - expect(screen.getByText('graylog_0')).toBeInTheDocument(); - expect( - within(screen.getByRole('columnheader', { name: 'Actions' })).getByRole('button', { name: /^delete all/i }), - ).toBeInTheDocument(); - }); - - it('offers archive-and-delete for managed indices only when archiving is available', () => { - mockIncompatibleIndices({ data: [graylogIndex] }); - const { rerender } = render(); - - expect(screen.getByRole('button', { name: /^archive and delete$/i })).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /archive and delete all/i })).not.toBeInTheDocument(); + expect(screen.getByText('Paginated incompatible indices')).toBeInTheDocument(); + expect(mockPaginatedEntityTable).toHaveBeenCalledTimes(1); - asMock(useCanArchive).mockReturnValue(false); - rerender(); - - expect(screen.queryByRole('button', { name: /archive and delete/i })).not.toBeInTheDocument(); - expect(screen.getByRole('button', { name: /^delete$/i })).toBeInTheDocument(); + const callProps = mockPaginatedEntityTable.mock.calls[0][0] as PaginatedEntityTableProps; + expect(callProps.fetchEntities).toBe(fetchIncompatibleIndices); + expect(callProps.keyFn).toBe(incompatibleIndicesKeyFn); + expect(callProps.entityAttributesAreCamelCase).toBe(false); + expect(callProps.tableLayout.entityTableId).toBe('incompatible_indices'); + expect(typeof callProps.entityActions).toBe('function'); + expect(callProps.bulkSelection.actions).toBeTruthy(); + expect(callProps.columnRenderers.attributes).toHaveProperty('index_name'); }); +}); - it('only offers reindex for system indices', () => { - mockIncompatibleIndices({ data: [systemIndex] }); - render(); +describe('createColumnRenderers', () => { + const renderers = createColumnRenderers(new Map(), new Set()); - expect(screen.getByRole('button', { name: /^reindex$/i })).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /reindex all/i })).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /delete/i })).not.toBeInTheDocument(); + it('falls back to "Unknown" for a missing version', () => { + expect(renderers.attributes.version.renderCell(undefined, makeIndex({ version: '' }), undefined)).toBe('Unknown'); }); - it('only offers delete for foreign indices', () => { - mockIncompatibleIndices({ data: [foreignIndex] }); - render(); - - expect(screen.getByRole('button', { name: /^delete$/i })).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /archive/i })).not.toBeInTheDocument(); - }); + it('renders the index name with status badges (not type badges)', () => { + const index = makeIndex({ index_name: 'graylog_2', warm_index: true, active_write_index: 'index-set-id' }); - it('only offers rotate for the active write index', () => { - mockIncompatibleIndices({ data: [writeIndex] }); - render(); + render(
{renderers.attributes.index_name.renderCell(index.index_name, index, undefined)}
); + expect(screen.getByText('graylog_2')).toBeInTheDocument(); expect(screen.getByText('active write index')).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /^rotate$/i })).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /delete/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /archive/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /reindex/i })).not.toBeInTheDocument(); - }); - - it('rotates the active write index via its index set and refreshes the list', async () => { - const sendTelemetry = jest.fn(); - asMock(useSendTelemetry).mockReturnValue(sendTelemetry); - const refetch = jest.fn(() => - Promise.resolve({ data: [makeIndex({ ...writeIndex, active_write_index: null })] }), - ) as unknown as ReturnType['refetch']; - mockIncompatibleIndices({ data: [writeIndex], refetch }); - render(); - - await userEvent.click(screen.getByRole('button', { name: /^rotate$/i })); - - const dialog = await screen.findByRole('dialog'); - expect(within(dialog).getByText(/is the active write index of its index set/i)).toBeInTheDocument(); - - await userEvent.click(within(dialog).getByRole('button', { name: /^rotate$/i })); - - await waitFor(() => expect(ClusterDeflector.cycleByindexSetId).toHaveBeenCalledWith('index-set-id')); - expect(sendTelemetry).toHaveBeenCalledWith( - TELEMETRY_EVENT_TYPE.DATANODE_OPENSEARCH_UPGRADE.WRITE_INDEX_ROTATE_CONFIRMED, - expect.objectContaining({ app_section: 'opensearch-upgrade' }), - ); - expect(UserNotification.success).toHaveBeenCalledWith(expect.stringContaining('graylog_2')); - expect(refetch).toHaveBeenCalled(); - }); - - it('excludes the active write index from bulk delete', () => { - mockIncompatibleIndices({ data: [graylogIndex, writeIndex] }); - render(); - - expect(screen.getByRole('button', { name: /delete all \(1\)/i })).toBeInTheDocument(); + expect(screen.queryByText('Warm')).not.toBeInTheDocument(); }); - it('uses the outdated delete endpoint for foreign indices', async () => { - const sendTelemetry = jest.fn(); - asMock(useSendTelemetry).mockReturnValue(sendTelemetry); - mockIncompatibleIndices({ data: [foreignIndex] }); - render(); - - await userEvent.click(screen.getByRole('button', { name: /^delete$/i })); - - const dialog = await screen.findByRole('dialog'); - expect(within(dialog).getByText(/this will permanently delete/i)).toBeInTheDocument(); - - await userEvent.click(within(dialog).getByRole('button', { name: /^delete$/i })); - - await waitFor(() => expect(IndexerIndices.deleteOutdated).toHaveBeenCalledWith('legacy-index')); - expect(sendTelemetry).toHaveBeenCalledWith( - TELEMETRY_EVENT_TYPE.DATANODE_OPENSEARCH_UPGRADE.INDEX_DELETE_CONFIRMED, - expect.objectContaining({ app_section: 'opensearch-upgrade' }), - ); - expect(UserNotification.success).toHaveBeenCalled(); - }); - - it('uses the generic delete endpoint for Graylog-managed indices', async () => { - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); + it('renders the primary type as a single badge in the category column', () => { + const cases: Array<[Partial, string]> = [ + [{ managed_index: true }, 'Graylog'], + [{ system_index: true }, 'System'], + [{}, 'Foreign'], + ]; - await userEvent.click(screen.getByRole('button', { name: /^delete$/i })); + cases.forEach(([overrides, expected]) => { + const { unmount } = render( +
{renderers.attributes.category.renderCell(undefined, makeIndex(overrides), undefined)}
, + ); - const dialog = await screen.findByRole('dialog'); - await userEvent.click(within(dialog).getByRole('button', { name: /^delete$/i })); - - await waitFor(() => expect(IndexerIndices.remove).toHaveBeenCalledWith('graylog_0')); - }); - - it('shows archive progress from the system job and hides row actions for a pending index', () => { - storePendingArchive('graylog_0', 'job-1'); - asMock(useClusterJobs).mockReturnValue({ - jobsById: new Map([['job-1', clusterJob({ job_status: 'running', percent_complete: 42 })]]), - jobsUpdatedAt: JOBS_POLLED_AFTER_ACTION, - }); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByText('42%')).toBeInTheDocument(); - expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '42'); - expect(screen.queryByRole('button', { name: /archive and delete/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /^delete$/i })).not.toBeInTheDocument(); - }); - - it('shows an "Archiving..." label without a bar until there is real progress', () => { - storePendingArchive('graylog_0', 'job-1'); - asMock(useClusterJobs).mockReturnValue({ - jobsById: new Map([['job-1', clusterJob({ job_status: 'running', percent_complete: 0 })]]), - jobsUpdatedAt: JOBS_POLLED_AFTER_ACTION, + expect(screen.getByText(expected)).toBeInTheDocument(); + unmount(); }); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByText(/archiving\.\.\./i)).toBeInTheDocument(); - expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); - }); - - it('shows a failed state and keeps row actions when the archive job errored', () => { - storePendingArchive('graylog_0', 'job-1'); - asMock(useClusterJobs).mockReturnValue({ - jobsById: new Map([['job-1', clusterJob({ job_status: 'error', info: 'Backend unreachable' })]]), - jobsUpdatedAt: JOBS_POLLED_AFTER_ACTION, - }); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByText(/archive failed/i)).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /^archive and delete$/i })).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /^delete$/i })).toBeInTheDocument(); - }); - - it('shows an archived-already badge when the job finished but the index is still incompatible', async () => { - storePendingArchive('graylog_0', 'job-1'); - asMock(useClusterJobs).mockReturnValue({ jobsById: new Map(), jobsUpdatedAt: JOBS_POLLED_AFTER_ACTION }); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); - expect(screen.getByText(/archived already/i)).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /^archive and delete$/i })).not.toBeInTheDocument(); - expect(screen.getByRole('button', { name: /^delete$/i })).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /delete all \(1\)/i })).toBeInTheDocument(); - await waitFor(() => - expect(JSON.parse(window.localStorage.getItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY))).toEqual([ - expect.objectContaining({ action: 'archive-delete', indexName: 'graylog_0', state: 'archived' }), - ]), - ); - }); - - it('flags an index the archive catalog already knows about, even without a local pending action', () => { - asMock(useArchivedIndexNames).mockReturnValue(new Set(['graylog_0'])); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByText(/archived already/i)).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /^archive and delete$/i })).not.toBeInTheDocument(); - expect(screen.getByRole('button', { name: /^delete$/i })).toBeInTheDocument(); - }); - - it('still counts an already-archived index as a bulk delete candidate', () => { - asMock(useArchivedIndexNames).mockReturnValue(new Set(['graylog_0'])); - mockIncompatibleIndices({ data: [graylogIndex, secondGraylogIndex] }); - render(); - - expect(screen.getByRole('button', { name: /delete all \(2\)/i })).toBeInTheDocument(); }); - it('keeps tracking when the jobs list predates the action (stale cache is not "job gone")', () => { - storePendingArchive('graylog_0', 'job-1'); - asMock(useClusterJobs).mockReturnValue({ jobsById: new Map(), jobsUpdatedAt: JOBS_POLLED_BEFORE_ACTION }); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByText(/archiving\.\.\./i)).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /archive and delete/i })).not.toBeInTheDocument(); - expect(window.localStorage.getItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY)).not.toBe('[]'); - }); + it('adds a Warm badge alongside the primary type', () => { + const index = makeIndex({ managed_index: true, warm_index: true }); - it('drops pending actions for indices that are no longer incompatible', async () => { - storePendingArchive('graylog_0', 'job-1'); - mockIncompatibleIndices({ data: [foreignIndex] }); - render(); + render(
{renderers.attributes.category.renderCell(undefined, index, undefined)}
); - expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); - await waitFor(() => expect(window.localStorage.getItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY)).toBe('[]')); + expect(screen.getByText('Graylog')).toBeInTheDocument(); + expect(screen.getByText('Warm')).toBeInTheDocument(); }); - it('keeps tracking an action without a system job id until its index disappears', () => { - storePendingArchive('graylog_0'); - asMock(useClusterJobs).mockReturnValue({ jobsById: new Map(), jobsUpdatedAt: JOBS_POLLED_AFTER_ACTION }); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByText(/archiving\.\.\./i)).toBeInTheDocument(); - expect(window.localStorage.getItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY)).not.toBe('[]'); - }); - - it('ignores malformed localStorage without crashing', () => { - window.localStorage.setItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY, JSON.stringify({ not: 'an array' })); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByRole('button', { name: /^archive and delete$/i })).toBeInTheDocument(); - expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); - }); - - it('drops invalid stored entries', () => { - window.localStorage.setItem( - PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY, - JSON.stringify([{ action: 'archive-delete' }, null, 'nonsense']), - ); - mockIncompatibleIndices({ data: [graylogIndex] }); - render(); - - expect(screen.getByRole('button', { name: /^archive and delete$/i })).toBeInTheDocument(); - expect(screen.queryByText(/archiving\.\.\./i)).not.toBeInTheDocument(); - }); - - it('runs bulk delete for eligible group indices and reports partial failures', async () => { - const refetch = jest.fn(() => - Promise.resolve({ data: [secondGraylogIndex] }), - ) as unknown as ReturnType['refetch']; - asMock(IndexerIndices.remove).mockImplementation((indexName: string) => - indexName === 'graylog_1' ? Promise.reject(new Error('Delete failed')) : Promise.resolve(), - ); - mockIncompatibleIndices({ data: [graylogIndex, secondGraylogIndex], refetch }); - render(); - - await userEvent.click(screen.getByRole('button', { name: /^delete all/i })); - - const dialog = await screen.findByRole('dialog'); - expect(within(dialog).getByText(/this will delete 2 incompatible indices/i)).toBeInTheDocument(); - - await userEvent.click(within(dialog).getByRole('button', { name: /delete all/i })); - - await waitFor(() => expect(IndexerIndices.remove).toHaveBeenCalledWith('graylog_0')); - expect(IndexerIndices.remove).toHaveBeenCalledWith('graylog_1'); - expect(UserNotification.warning).toHaveBeenCalledWith( - expect.stringContaining('1 succeeded, 1 failed'), - 'Some indices could not be deleted', - ); - expect(refetch).toHaveBeenCalled(); - }); - - it('tracks a successful archive-and-delete job as a pending action', async () => { - const binding = archiveBinding(); - asMock(useIndexArchive).mockReturnValue(binding); - mockIncompatibleIndices({ data: [graylogIndex, secondGraylogIndex] }); - render(); - - await userEvent.click(screen.getAllByRole('button', { name: /^archive and delete$/i })[0]); - - const dialog = await screen.findByRole('dialog'); - await userEvent.click(within(dialog).getByRole('button', { name: /^archive and delete$/i })); - - await waitFor(() => expect(binding.archiveAndDeleteIndex).toHaveBeenCalledWith('graylog_0')); - - const storedActions = JSON.parse(window.localStorage.getItem(PENDING_INCOMPATIBLE_INDEX_ACTIONS_STORAGE_KEY)); - - expect(storedActions).toEqual([ - expect.objectContaining({ action: 'archive-delete', indexName: 'graylog_0', systemJobId: 'archive-job-id' }), - ]); - expect(UserNotification.success).toHaveBeenCalledWith('Archive and delete job for "graylog_0" was started.'); - }); - - it('skips in-progress archive actions when running a bulk action', async () => { - storePendingArchive('graylog_0', 'job-1'); - asMock(useClusterJobs).mockReturnValue({ - jobsById: new Map([['job-1', clusterJob({ job_status: 'running', percent_complete: 42 })]]), - jobsUpdatedAt: JOBS_POLLED_AFTER_ACTION, - }); - mockIncompatibleIndices({ data: [graylogIndex, secondGraylogIndex] }); - render(); - - await userEvent.click(screen.getByRole('button', { name: /^delete all/i })); - - const dialog = await screen.findByRole('dialog'); - expect(within(dialog).getByText(/this will delete 1 incompatible index/i)).toBeInTheDocument(); - - await userEvent.click(within(dialog).getByRole('button', { name: /delete all/i })); + it('renders an em dash for an unknown message range', () => { + render(
{renderers.attributes.begin.renderCell(null, makeIndex({}), undefined)}
); - await waitFor(() => expect(IndexerIndices.remove).toHaveBeenCalledTimes(1)); - expect(IndexerIndices.remove).toHaveBeenCalledWith('graylog_1'); - expect(IndexerIndices.remove).not.toHaveBeenCalledWith('graylog_0'); + expect(screen.getByText('—')).toBeInTheDocument(); }); }); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx index 25599d33df9c..c2bb793acff4 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx @@ -14,41 +14,24 @@ * along with this program. If not, see * . */ -import React, { useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import styled, { css } from 'styled-components'; +import { useQueryClient } from '@tanstack/react-query'; -import { Alert, Button, SegmentedControl } from 'components/bootstrap'; -import { ConfirmDialog, Spinner } from 'components/common'; +import { PaginatedEntityTable } from 'components/common'; import useCanArchive from 'components/indices/hooks/useCanArchive'; -import useIncompatibleIndices from 'components/indices/hooks/useIncompatibleIndices'; -import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; -import extractErrorMessage from 'util/extractErrorMessage'; -import UserNotification from 'util/UserNotification'; -import BulkIndexActionConfirmDialog from './BulkIndexActionConfirmDialog'; -import { getBulkIndexActionCandidates, getBulkIndexActionNotification, runBulkIndexAction } from './bulkIndexActions'; -import type { BulkIndexActionCandidate, BulkIndexActionNotification } from './bulkIndexActions'; -import IndicesGroupTable from './IndicesGroupTable'; +import { + fetchIncompatibleIndices, + incompatibleIndicesKeyFn, + INCOMPATIBLE_INDICES_QUERY_KEY, +} from './fetchIncompatibleIndices'; +import type { IncompatibleIndexRow, IncompatibleIndicesResponse } from './fetchIncompatibleIndices'; +import { createColumnRenderers, DEFAULT_DISPLAYED_COLUMNS } from './IncompatibleIndicesColumnRenderers'; +import IncompatibleIndexTableActions from './IncompatibleIndexTableActions'; +import IncompatibleIndicesBulkActions from './IncompatibleIndicesBulkActions'; import useArchivedIndexNames from './hooks/useArchivedIndexNames'; import usePendingIncompatibleIndexActions from './hooks/usePendingIncompatibleIndexActions'; -import { useIncompatibleIndexActionDefinitions } from './incompatibleIndexActions'; -import type { ConfirmedAction } from './incompatibleIndexActions'; -import { getFirstGroupWithIndices, getSelectedGroup, groupIncompatibleIndices } from './incompatibleIndexGroups'; - -const TELEMETRY_DEFAULTS = { app_pathname: 'datanode', app_section: 'opensearch-upgrade' } as const; - -const showBulkNotification = ({ type, message, title }: BulkIndexActionNotification) => { - const notify = (notification: (notificationMessage: string, notificationTitle?: string) => void) => - title ? notification(message, title) : notification(message); - - if (type === 'success') { - notify(UserNotification.success); - } else if (type === 'warning') { - notify(UserNotification.warning); - } else { - notify(UserNotification.error); - } -}; const Heading = styled.h4( ({ theme }) => css` @@ -57,221 +40,100 @@ const Heading = styled.h4( `, ); -const ActionConfirmDialog = ({ - confirmedAction, - isSubmitting, - onCancel, - onConfirm, -}: { - confirmedAction: ConfirmedAction; - isSubmitting: boolean; - onCancel: () => void; - onConfirm: () => void; -}) => { - const actionDefinitions = useIncompatibleIndexActionDefinitions(); - const actionDefinition = actionDefinitions[confirmedAction.action]; - - return ( - - {actionDefinition.confirmationBody(confirmedAction.index)} - - ); +const TABLE_LAYOUT = { + entityTableId: 'incompatible_indices', + defaultSort: { attributeId: 'index_name', direction: 'asc' as const }, + defaultDisplayedAttributes: DEFAULT_DISPLAYED_COLUMNS, + defaultPageSize: 20, + defaultColumnOrder: ['index_name', 'category', 'version', 'begin', 'end'], }; const IncompatibleIndicesTable = () => { - const { data: incompatibleIndices, isError, isLoading, refetch } = useIncompatibleIndices(); + const queryClient = useQueryClient(); const canArchive = useCanArchive(); - const actionDefinitions = useIncompatibleIndexActionDefinitions(); - const sendTelemetry = useSendTelemetry(); + const [loadedIndices, setLoadedIndices] = useState>([]); + const [hasLoaded, setHasLoaded] = useState(false); + + const refetch = useCallback( + () => queryClient.invalidateQueries({ queryKey: INCOMPATIBLE_INDICES_QUERY_KEY }), + [queryClient], + ); + + const incompatibleIndexNames = useMemo(() => loadedIndices.map((index) => index.index_name), [loadedIndices]); + const archivedIndexNames = useArchivedIndexNames(incompatibleIndexNames, canArchive); const { pendingIndexStatuses, addArchiveDeleteAction, isArchiveJobRunning, refetchClusterJobs } = usePendingIncompatibleIndexActions({ - incompatibleIndices, - isLoading, - isError, + incompatibleIndices: loadedIndices, + isLoading: !hasLoaded, + isError: false, refetch, canArchive, }); - const incompatibleIndexNames = incompatibleIndices.map((index) => index.index_name); - const archivedIndexNames = useArchivedIndexNames(incompatibleIndexNames, canArchive); - const [confirmedAction, setConfirmedAction] = useState(); - const [confirmedBulkAction, setConfirmedBulkAction] = useState(); - const [isSubmitting, setIsSubmitting] = useState(false); - const [isBulkSubmitting, setIsBulkSubmitting] = useState(false); - const [selectedGroupId, setSelectedGroupId] = useState(); - const indicesGroups = groupIncompatibleIndices(incompatibleIndices); - const firstGroupWithIndices = getFirstGroupWithIndices(indicesGroups); - const activeGroupId = selectedGroupId ?? firstGroupWithIndices; - const selectedGroup = getSelectedGroup(indicesGroups, activeGroupId); + // Suppress archive actions while an archive job runs — the backend archive job is single-concurrency. const archiveActionsAvailable = canArchive && !isArchiveJobRunning; - const segments = indicesGroups.map((group) => ({ - value: group.id, - label: `${group.shortLabel} (${group.indices.length})`, - })); - const bulkActions = getBulkIndexActionCandidates({ - indices: selectedGroup.indices, - canArchive: archiveActionsAvailable, - pendingIndexStatuses, - archivedIndexNames, - }); - - const closeConfirmDialog = () => setConfirmedAction(undefined); - const closeBulkConfirmDialog = () => setConfirmedBulkAction(undefined); - const cancelBulkConfirmDialog = () => { - if (!isBulkSubmitting) { - closeBulkConfirmDialog(); - } - }; - - const finalizeAfterActions = async () => { - const { data: updatedIncompatibleIndices = [] } = await refetch(); - const updatedGroups = groupIncompatibleIndices(updatedIncompatibleIndices); - const updatedSelectedGroup = getSelectedGroup(updatedGroups, activeGroupId); - - if (updatedSelectedGroup.indices.length === 0) { - setSelectedGroupId(getFirstGroupWithIndices(updatedGroups)); - } - }; - - const showArchiveJobConflictWarning = () => { - UserNotification.warning( - 'Another archive job is already running. New archive jobs can be started after it finishes.', - 'Archive job already running', - ); - refetchClusterJobs?.(); - }; - - const handleConfirm = async () => { - if (!confirmedAction) { - return; - } - - const { action, index } = confirmedAction; - const actionDefinition = actionDefinitions[action]; - sendTelemetry(actionDefinition.telemetryEventType, { ...TELEMETRY_DEFAULTS }); - setIsSubmitting(true); + const handleDataLoaded = useCallback((data: IncompatibleIndicesResponse) => { + setLoadedIndices(data.list); + setHasLoaded(true); + }, []); - try { - const actionResponse = await actionDefinition.run(index); - const pendingArchive = actionDefinition.getPendingArchiveTracking?.(index, actionResponse); - - if (pendingArchive) { - addArchiveDeleteAction(pendingArchive); - } - - UserNotification.success(actionDefinition.successMessage(index)); - await finalizeAfterActions(); - closeConfirmDialog(); - } catch (errorThrown) { - const errorMessage = extractErrorMessage(errorThrown); - - if (actionDefinition.isArchiveJobConflict?.(errorMessage)) { - showArchiveJobConflictWarning(); - closeConfirmDialog(); - } else { - UserNotification.error(errorMessage, `Could not ${actionDefinition.confirmText.toLowerCase()}.`); - } - } finally { - setIsSubmitting(false); - } - }; - - const handleBulkConfirm = async () => { - if (!confirmedBulkAction || isBulkSubmitting) { - return; - } - - sendTelemetry(actionDefinitions[confirmedBulkAction.action].telemetryEventType, { - ...TELEMETRY_DEFAULTS, - app_action_value: 'bulk', - bulk_count: confirmedBulkAction.targetIndices.length, - }); - setIsBulkSubmitting(true); - - try { - const result = await runBulkIndexAction({ - action: confirmedBulkAction.action, - indices: confirmedBulkAction.targetIndices, - }); + const columnRenderers = useMemo( + () => createColumnRenderers(pendingIndexStatuses, archivedIndexNames), + [pendingIndexStatuses, archivedIndexNames], + ); - showBulkNotification(getBulkIndexActionNotification(confirmedBulkAction, result)); - await finalizeAfterActions(); - closeBulkConfirmDialog(); - } catch (errorThrown) { - UserNotification.error( - extractErrorMessage(errorThrown), - `Could not ${confirmedBulkAction.confirmText.toLowerCase()}.`, + const renderActions = useCallback( + (index: IncompatibleIndexRow) => { + const pendingStatus = pendingIndexStatuses.get(index.index_name); + const isArchived = + pendingStatus?.state !== 'archiving' && + (archivedIndexNames.has(index.index_name) || pendingStatus?.state === 'archived'); + + return ( + ); - } finally { - setIsBulkSubmitting(false); - } - }; - - if (isLoading) { - return ; - } - - if (isError) { - return ( - - Could not load incompatible indices — retrying automatically.{' '} - - - ); - } + }, + [pendingIndexStatuses, archivedIndexNames, archiveActionsAvailable, addArchiveDeleteAction, refetchClusterJobs, refetch], + ); - if (!incompatibleIndices.length) { - return No incompatible indices found.; - } + const bulkSelection = useMemo( + () => ({ + actions: ( + + ), + }), + [loadedIndices, archiveActionsAvailable, pendingIndexStatuses, archivedIndexNames, refetch], + ); return ( <> Incompatible indices - + humanName="incompatible indices" + tableLayout={TABLE_LAYOUT} + fetchEntities={fetchIncompatibleIndices} + keyFn={incompatibleIndicesKeyFn} + columnRenderers={columnRenderers} + entityActions={renderActions} + bulkSelection={bulkSelection} + onDataLoaded={handleDataLoaded} + entityAttributesAreCamelCase={false} /> - - - {confirmedAction && ( - - )} - {confirmedBulkAction && ( - - )} ); }; diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IndicesGroupTable.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IndicesGroupTable.tsx deleted file mode 100644 index a240ffa29edf..000000000000 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IndicesGroupTable.tsx +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -import React from 'react'; -import styled, { css } from 'styled-components'; - -import { Alert, Button, ButtonToolbar, Label, Table } from 'components/bootstrap'; -import { ProgressBar } from 'components/common'; -import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; - -import type { BulkIndexActionCandidate } from './bulkIndexActions'; -import type { PendingIndexStatus } from './hooks/usePendingIncompatibleIndexActions'; -import { CORE_ACTION_DEFINITIONS, getAvailableActions, useIncompatibleIndexActionDefinitions } from './incompatibleIndexActions'; -import type { ConfirmedAction } from './incompatibleIndexActions'; -import type { IndicesGroup } from './incompatibleIndexGroups'; - -const indexNameBadges = (index: IncompatibleIndex, isArchived: boolean) => - [ - { text: 'warm', style: 'default', show: index.warm_index }, - { text: 'active write index', style: 'default', show: !!index.active_write_index }, - { text: 'archived already', style: 'success', show: isArchived }, - ] as const; - -const ActionsToolbar = styled(ButtonToolbar)` - justify-content: flex-end; -`; - -const BulkActionsToolbar = styled(ButtonToolbar)( - () => css` - justify-content: flex-end; - margin: 0; - `, -); - -const ArchiveProgressBar = styled(ProgressBar)` - display: inline-flex; - width: 120px; - margin-bottom: 0; - vertical-align: middle; -`; - -const ScrollableTableWrapper = styled.div( - ({ theme }) => css` - margin-top: ${theme.spacings.md}; - margin-bottom: ${theme.spacings.md}; - - & > table { - margin-bottom: 0; - table-layout: fixed; - } - - & thead, - & tbody { - display: block; - } - - & thead tr, - & tbody tr { - display: table; - width: 100%; - table-layout: fixed; - } - - & tbody { - max-height: 300px; - overflow-y: auto; - scrollbar-gutter: stable; - } - - & thead { - scrollbar-gutter: stable; - overflow-y: hidden; - } - - & thead th { - background-color: ${theme.colors.table.head.background}; - } - - & tr > *:nth-child(1) { - width: 40%; - text-align: left; - } - - & tr > *:nth-child(2) { - width: 30%; - text-align: left; - } - - & tr > *:nth-child(3) { - width: 30%; - text-align: right; - } - `, -); - -const IncompatibleIndexActions = ({ - index, - onAction, - canArchive, - pendingStatus, - isArchived, -}: { - index: IncompatibleIndex; - onAction: (action: ConfirmedAction) => void; - canArchive: boolean; - pendingStatus: PendingIndexStatus | undefined; - isArchived: boolean; -}) => { - const actionDefinitions = useIncompatibleIndexActionDefinitions(); - - if (pendingStatus?.state === 'archiving') { - // No empty 0% bar flash for jobs that finish almost instantly. - return pendingStatus.percent > 0 ? ( - - ) : ( - - ); - } - - const actions = getAvailableActions(index, canArchive, isArchived); - - return ( - - {pendingStatus?.state === 'failed' && ( - - )} - {actions.map((action) => { - const actionDefinition = actionDefinitions[action]; - - return ( - - ); - })} - - ); -}; - -const IndicesGroupTable = ({ - group, - onAction, - onBulkAction, - canArchive, - pendingIndexStatuses, - archivedIndexNames, - bulkActions, - isBulkActionSubmitting, -}: { - group: IndicesGroup; - onAction: (action: ConfirmedAction) => void; - onBulkAction: (bulkAction: BulkIndexActionCandidate) => void; - canArchive: boolean; - pendingIndexStatuses: Map; - archivedIndexNames: ReadonlySet; - bulkActions: Array; - isBulkActionSubmitting: boolean; -}) => { - if (group.indices.length === 0) { - return No incompatible {group.shortLabel} indices.; - } - - return ( - - - - - - - - - - - {group.indices.map((index) => { - const pendingStatus = pendingIndexStatuses.get(index.index_name); - const isArchived = - pendingStatus?.state !== 'archiving' && - (archivedIndexNames.has(index.index_name) || pendingStatus?.state === 'archived'); - - return ( - - - - - - ); - })} - -
{group.indexLabel}OpenSearch version - {bulkActions.length > 0 && ( - - {bulkActions.map((bulkAction) => ( - - ))} - - )} -
- {index.index_name} - {indexNameBadges(index, isArchived) - .filter((badge) => badge.show) - .map(({ text, style }) => ( - -   - - - ))} - {index.version || 'Unknown'} - -
- - ); -}; - -export default IndicesGroupTable; diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts new file mode 100644 index 000000000000..563e5347ea03 --- /dev/null +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import { SystemIndexerIndices } from '@graylog/server-api'; + +import FiltersForQueryParams from 'components/common/EntityFilters/FiltersForQueryParams'; +import type { Attribute, SearchParams } from 'stores/PaginationTypes'; +import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; + +// Entity tables require an `id` on every row; the index name is a stable natural key. +export type IncompatibleIndexRow = IncompatibleIndex & { id: string }; + +export type IncompatibleIndicesResponse = { + list: Array; + pagination: { total: number }; + attributes: Array; +}; + +export const INCOMPATIBLE_INDICES_QUERY_KEY = ['incompatibleIndices'] as const; + +type ListOutdatedIndicesSort = Parameters[0]; +type ListOutdatedIndicesOrder = Parameters[4]; + +export const fetchIncompatibleIndices = (searchParams: SearchParams): Promise => { + // This endpoint is backed by an in-memory Lucene search that only understands the `query` string + // (there is no separate `filters` param), so fold the active filter chips into the query. + const query = [searchParams.query, ...FiltersForQueryParams(searchParams.filters)].filter(Boolean).join(' '); + const sort = (searchParams.sort?.attributeId ?? 'index_name') as ListOutdatedIndicesSort; + const order = (searchParams.sort?.direction ?? 'asc') as ListOutdatedIndicesOrder; + + return SystemIndexerIndices.listOutdatedIndices(sort, searchParams.page, searchParams.pageSize, query, order).then( + ({ elements, attributes, total }) => ({ + list: elements.map((element) => ({ ...element, id: element.index_name })) as Array, + pagination: { total }, + attributes, + }), + ); +}; + +export const incompatibleIndicesKeyFn = (searchParams?: SearchParams) => [ + ...INCOMPATIBLE_INDICES_QUERY_KEY, + 'paginated', + ...(searchParams ? [searchParams] : []), +]; diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx index 576449acc88a..e8a2a553276b 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx @@ -16,7 +16,7 @@ */ import React from 'react'; -import { ClusterDeflector, IndexerIndices } from '@graylog/server-api'; +import { ClusterDeflector, IndexerIndices, SystemIndexerIndices } from '@graylog/server-api'; import type { StyleProps } from 'components/bootstrap/Button'; import type { IndexArchiveBinding } from 'components/indices/archive/types'; @@ -51,9 +51,9 @@ type ActionDefinition = { }; const deleteIncompatibleIndex = (index: IncompatibleIndex) => - index.managed_index ? IndexerIndices.remove(index.index_name) : IndexerIndices.deleteOutdated(index.index_name); + index.managed_index ? IndexerIndices.remove(index.index_name) : SystemIndexerIndices.deleteOutdated(index.index_name); -const reindexSystemIndex = (index: IncompatibleIndex) => IndexerIndices.reindex(index.index_name); +const reindexSystemIndex = (index: IncompatibleIndex) => SystemIndexerIndices.reindex(index.index_name); const rotateWriteIndex = (index: IncompatibleIndex) => ClusterDeflector.cycleByindexSetId(index.active_write_index); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.test.ts b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.test.ts deleted file mode 100644 index f437de5579f5..000000000000 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; - -import { groupIncompatibleIndices, getFirstGroupWithIndices, getSelectedGroup } from './incompatibleIndexGroups'; - -const makeIndex = (overrides: Partial): IncompatibleIndex => ({ - index_name: 'index', - version: '2.0.0', - warm_index: false, - managed_index: false, - system_index: false, - active_write_index: null, - ...overrides, -}); - -const graylogIndex = makeIndex({ index_name: 'graylog_0', managed_index: true }); -const systemIndex = makeIndex({ index_name: '.system', system_index: true }); -const foreignIndex = makeIndex({ index_name: 'legacy' }); - -describe('incompatibleIndexGroups', () => { - describe('groupIncompatibleIndices', () => { - it('returns the three groups in a stable order', () => { - expect(groupIncompatibleIndices([]).map((group) => group.id)).toEqual(['graylog', 'system', 'foreign']); - }); - - it('classifies each index into its matching group', () => { - const [graylog, system, foreign] = groupIncompatibleIndices([graylogIndex, systemIndex, foreignIndex]); - - expect(graylog.indices).toEqual([graylogIndex]); - expect(system.indices).toEqual([systemIndex]); - expect(foreign.indices).toEqual([foreignIndex]); - }); - - it('treats a managed system index as a system index', () => { - const managedSystemIndex = makeIndex({ index_name: '.managed-system', managed_index: true, system_index: true }); - const [graylog, system] = groupIncompatibleIndices([managedSystemIndex]); - - expect(graylog.indices).toEqual([]); - expect(system.indices).toEqual([managedSystemIndex]); - }); - }); - - describe('getFirstGroupWithIndices', () => { - it('returns the id of the first non-empty group', () => { - const groups = groupIncompatibleIndices([systemIndex]); - - expect(getFirstGroupWithIndices(groups)).toBe('system'); - }); - - it('falls back to the default group when every group is empty', () => { - const groups = groupIncompatibleIndices([]); - - expect(getFirstGroupWithIndices(groups)).toBe('graylog'); - }); - }); - - describe('getSelectedGroup', () => { - it('returns the group matching the given id', () => { - const groups = groupIncompatibleIndices([foreignIndex]); - - expect(getSelectedGroup(groups, 'foreign').id).toBe('foreign'); - }); - - it('falls back to the first group for an unknown id', () => { - const groups = groupIncompatibleIndices([]); - - expect(getSelectedGroup(groups, 'does-not-exist').id).toBe('graylog'); - }); - }); -}); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.ts b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.ts deleted file mode 100644 index 10f3aa34b0cc..000000000000 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexGroups.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; - -type IndexGroupId = 'graylog' | 'system' | 'foreign'; - -type IndexGroupDefinition = { - id: IndexGroupId; - shortLabel: string; - indexLabel: string; - matches: (index: IncompatibleIndex) => boolean; -}; - -export type IndicesGroup = Omit & { - indices: Array; -}; - -const INDEX_GROUPS: Array = [ - { - id: 'graylog', - shortLabel: 'Graylog', - indexLabel: 'Graylog index', - matches: (index) => index.managed_index && !index.system_index, - }, - { - id: 'system', - shortLabel: 'System', - indexLabel: 'System index', - matches: (index) => index.system_index, - }, - { - id: 'foreign', - shortLabel: 'Foreign', - indexLabel: 'Foreign index', - matches: (index) => !index.managed_index && !index.system_index, - }, -]; - -const DEFAULT_GROUP_ID = INDEX_GROUPS[0].id; - -export const groupIncompatibleIndices = (indices: Array): Array => - INDEX_GROUPS.map(({ id, shortLabel, indexLabel, matches }) => ({ - id, - shortLabel, - indexLabel, - indices: indices.filter(matches), - })); - -export const getFirstGroupWithIndices = (groups: Array) => - groups.find((group) => group.indices.length > 0)?.id ?? DEFAULT_GROUP_ID; - -export const getSelectedGroup = (groups: Array, selectedGroupId: string) => - groups.find((group) => group.id === selectedGroupId) ?? groups[0]; diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/telemetry.ts b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/telemetry.ts new file mode 100644 index 000000000000..abdb47016860 --- /dev/null +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/telemetry.ts @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +export const TELEMETRY_DEFAULTS = { app_pathname: 'datanode', app_section: 'opensearch-upgrade' } as const; diff --git a/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts b/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts index 78347410781d..05f1e2a81c88 100644 --- a/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts +++ b/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts @@ -16,7 +16,7 @@ */ import { useQuery } from '@tanstack/react-query'; -import { IndexerIndices } from '@graylog/server-api'; +import { SystemIndexerIndices } from '@graylog/server-api'; export type IncompatibleIndex = { index_name: string; @@ -26,6 +26,10 @@ export type IncompatibleIndex = { system_index: boolean; /** Id of the index set this index is the active write index of, `null` for all other indices. */ active_write_index: string | null; + /** Start of the message time range stored in the index, `null` when unknown / not calculated. */ + begin?: string | null; + /** End of the message time range stored in the index, `null` when unknown / not calculated. */ + end?: string | null; }; const ERROR_REFETCH_INTERVAL_MS = 30000; @@ -39,7 +43,7 @@ const useIncompatibleIndices = () => { } = useQuery({ queryKey: ['incompatibleIndices'], // No error toast: the panels render a persistent error state, and background retries would spam toasts. - queryFn: () => IndexerIndices.getOutdatedIndices() as Promise>, + queryFn: () => SystemIndexerIndices.getOutdatedIndices() as Promise>, retry: 2, refetchInterval: (query) => (query.state.status === 'error' ? ERROR_REFETCH_INTERVAL_MS : false), }); From e772629e5b4b5fdefae28db02af530de8368786a Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Fri, 17 Jul 2026 13:14:41 +0200 Subject: [PATCH 6/9] add bulk delete functionality --- .../indexer/indices/OutdatedIndexService.java | 8 +- .../system/indexer/OutdatedIndexResource.java | 72 +++++++++++++++++- .../indices/OutdatedIndexServiceTest.java | 4 + .../indexer/OutdatedIndicesResourceTest.java | 54 +++++++++++++ .../IncompatibleIndicesBulkActions.tsx | 75 ++++++++++++++----- 5 files changed, 193 insertions(+), 20 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java index 92166fc85b29..b47b2f785e59 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/OutdatedIndexService.java @@ -19,12 +19,14 @@ import com.github.zafarkhaja.semver.ParseException; import com.github.zafarkhaja.semver.Version; +import com.google.common.eventbus.EventBus; import jakarta.inject.Inject; import jakarta.inject.Singleton; import jakarta.validation.constraints.NotNull; import org.graylog2.indexer.ElasticsearchException; import org.graylog2.indexer.cluster.Cluster; import org.graylog2.indexer.indexset.registry.IndexSetRegistry; +import org.graylog2.indexer.indices.events.IndicesDeletedEvent; import org.graylog2.indexer.ranges.IndexRange; import org.graylog2.indexer.ranges.IndexRangeService; import org.graylog2.indexer.security.IndexerAdminCert; @@ -51,14 +53,16 @@ public class OutdatedIndexService { private final IndicesAdapter indicesAdapter; private final IndexSetRegistry indexSetRegistry; private final IndexRangeService indexRangeService; + private final EventBus eventBus; private final Cluster cluster; @Inject public OutdatedIndexService(@IndexerAdminCert IndicesAdapter indicesAdapter, IndexSetRegistry indexSetRegistry, - IndexRangeService indexRangeService, Cluster cluster) { + IndexRangeService indexRangeService, EventBus eventBus, Cluster cluster) { this.indicesAdapter = indicesAdapter; this.indexSetRegistry = indexSetRegistry; this.indexRangeService = indexRangeService; + this.eventBus = eventBus; this.cluster = cluster; } @@ -183,5 +187,7 @@ private Map cleanIndexSettings(Map settings, boo public void delete(@NotNull String index) { indicesAdapter.delete(index); + // Mirror Indices#delete so listeners clean up index ranges and cached field types for managed indices. + eventBus.post(IndicesDeletedEvent.create(index)); } } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java index e57a32e45e4e..36499c34c37f 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/indexer/OutdatedIndexResource.java @@ -25,6 +25,8 @@ import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.inject.Inject; import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; @@ -38,11 +40,17 @@ import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog2.audit.AuditActor; +import org.graylog2.audit.AuditEventSender; import org.graylog2.audit.AuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; +import org.graylog2.audit.jersey.NoAuditEvent; import org.graylog2.database.PaginatedList; import org.graylog2.indexer.indices.OutdatedIndex; import org.graylog2.indexer.indices.OutdatedIndexService; +import org.graylog2.rest.bulk.model.BulkOperationFailure; +import org.graylog2.rest.bulk.model.BulkOperationRequest; +import org.graylog2.rest.bulk.model.BulkOperationResponse; import org.graylog2.rest.models.SortOrder; import org.graylog2.rest.models.tools.responses.PageListResponse; import org.graylog2.rest.resources.entities.EntityAttribute; @@ -58,10 +66,14 @@ import org.graylog2.utilities.lucene.LuceneInMemorySearchEngine; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; @Tag(name = "System/Indexer/Indices", description = "Outdated index discovery") @RequiresAuthentication @@ -93,12 +105,14 @@ private static Set categoryFilterOptions() { .build(); private final OutdatedIndexService outdatedIndexService; + private final AuditEventSender auditEventSender; private final InMemorySearchEngine outdatedIndexSearchService; private final SearchQueryParser searchQueryParser; @Inject - public OutdatedIndexResource(OutdatedIndexService outdatedIndexService) { + public OutdatedIndexResource(OutdatedIndexService outdatedIndexService, AuditEventSender auditEventSender) { this.outdatedIndexService = outdatedIndexService; + this.auditEventSender = auditEventSender; final Supplier> cachingSupplier = Suppliers.memoizeWithExpiration( outdatedIndexService::getOutdatedIndices, 10, @@ -173,4 +187,60 @@ public void deleteOutdated(@Parameter(name = "index") @PathParam("index") @NotNu .findAny().orElseThrow(() -> new NotFoundException("Index " + index + " not found or is an index managed by Graylog")); outdatedIndexService.delete(outdatedIndex.indexName()); } + + @POST + @Path("/bulk_delete") + @Consumes(MediaType.APPLICATION_JSON) + @Timed + @Operation(summary = "Delete a bulk of outdated indices") + @NoAuditEvent("Each deleted index is audited individually below") + public BulkOperationResponse bulkDeleteOutdated(@Parameter(name = "Entities to remove", required = true) BulkOperationRequest request) { + if (request == null || request.entityIds() == null || request.entityIds().isEmpty()) { + throw new BadRequestException("No index names provided"); + } + + final Map outdatedByName = getOutdatedIndices().stream() + .collect(Collectors.toMap(OutdatedIndex::indexName, Function.identity(), (existing, replacement) -> existing)); + + final List failures = new ArrayList<>(); + int deleted = 0; + for (String index : request.entityIds()) { + final String failure = deleteSingleOutdated(index, outdatedByName.get(index)); + if (failure != null) { + failures.add(new BulkOperationFailure(index, failure)); + } else { + deleted++; + } + } + return new BulkOperationResponse(deleted, failures); + } + + private String deleteSingleOutdated(String index, OutdatedIndex outdatedIndex) { + if (outdatedIndex == null) { + return "Index " + index + " not found or is not an outdated index"; + } + if (outdatedIndex.activeWriteIndex() != null) { + return "Index " + index + " is the active write index and cannot be deleted"; + } + // Managed indices are gated by a per-index permission (see IndicesResource#delete); non-managed/foreign + // indices have no per-index grant and are only gated by the general permission (see #deleteOutdated). + final boolean permitted = outdatedIndex.managedIndex() + ? isPermitted(RestPermissions.INDICES_DELETE, index) + : isPermitted(RestPermissions.INDICES_DELETE); + if (!permitted) { + return "Not authorized to delete index " + index; + } + try { + outdatedIndexService.delete(index); + auditEventSender.success(auditActor(), AuditEventTypes.ES_INDEX_DELETE, Map.of("index_name", index)); + return null; + } catch (Exception e) { + auditEventSender.failure(auditActor(), AuditEventTypes.ES_INDEX_DELETE, Map.of("index_name", index)); + return e.getMessage(); + } + } + + private AuditActor auditActor() { + return AuditActor.user(getSubject().getPrincipal().toString()); + } } diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java index 75bd5fa84f4a..53e19343d68b 100644 --- a/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java +++ b/graylog2-server/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceTest.java @@ -17,6 +17,7 @@ package org.graylog2.indexer.indices; +import com.google.common.eventbus.EventBus; import org.assertj.core.api.Assertions; import org.bson.conversions.Bson; import org.graylog2.indexer.cluster.Cluster; @@ -66,6 +67,9 @@ class OutdatedIndexServiceTest { @Mock IndexRangeService indexRangeService; + @Mock + EventBus eventBus; + @InjectMocks OutdatedIndexService outdatedIndexService; diff --git a/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java b/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java index 9ec443423248..d396185208de 100644 --- a/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java +++ b/graylog2-server/src/test/java/org/graylog2/rest/resources/system/indexer/OutdatedIndicesResourceTest.java @@ -19,11 +19,15 @@ import jakarta.ws.rs.ForbiddenException; import org.assertj.core.api.Assertions; +import org.graylog2.audit.AuditEventSender; import org.graylog2.indexer.NodeInfoCache; import org.graylog2.indexer.indexset.registry.IndexSetRegistry; import org.graylog2.indexer.indices.Indices; import org.graylog2.indexer.indices.OutdatedIndex; import org.graylog2.indexer.indices.OutdatedIndexService; +import org.graylog2.rest.bulk.model.BulkOperationFailure; +import org.graylog2.rest.bulk.model.BulkOperationRequest; +import org.graylog2.rest.bulk.model.BulkOperationResponse; import org.graylog2.security.WithAuthorization; import org.graylog2.security.WithAuthorizationExtension; import org.junit.jupiter.api.Test; @@ -35,6 +39,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -55,6 +60,9 @@ class OutdatedIndicesResourceTest { @Mock OutdatedIndexService outdatedIndexService; + @Mock + AuditEventSender auditEventSender; + @InjectMocks OutdatedIndexResource outdatedIndexResource; @@ -90,4 +98,50 @@ void reindexSucceeds() { verify(outdatedIndexService, times(1)).reindex(".outdated1", true); } + @Test + @WithAuthorization(permissions = {"indices:read", "indices:delete"}) + void bulkDeleteOutdatedDeletesEligibleIndicesAndReportsFailures() { + when(outdatedIndexService.getOutdatedIndices()).thenReturn(List.of( + new OutdatedIndex("foreign1", "1.3.0", false, false, null), + new OutdatedIndex("managed1", "1.3.0", false, true, null), + new OutdatedIndex("writeindex", "1.3.0", false, true, "id1") + )); + + final BulkOperationResponse response = outdatedIndexResource.bulkDeleteOutdated( + new BulkOperationRequest(List.of("foreign1", "managed1", "writeindex", "missing"))); + + verify(outdatedIndexService).delete("foreign1"); + verify(outdatedIndexService).delete("managed1"); + verify(outdatedIndexService, never()).delete("writeindex"); + verify(outdatedIndexService, never()).delete("missing"); + assertThat(response.successfullyPerformed()).isEqualTo(2); + assertThat(response.failures()).extracting(BulkOperationFailure::entityId) + .containsExactlyInAnyOrder("writeindex", "missing"); + } + + @Test + @WithAuthorization(permissions = {"indices:read", "indices:delete:managed1"}) + void bulkDeleteOutdatedUsesPerIndexPermissionForManagedAndGeneralForForeign() { + when(outdatedIndexService.getOutdatedIndices()).thenReturn(List.of( + new OutdatedIndex("managed1", "1.3.0", false, true, null), + new OutdatedIndex("foreign1", "1.3.0", false, false, null) + )); + + final BulkOperationResponse response = outdatedIndexResource.bulkDeleteOutdated( + new BulkOperationRequest(List.of("managed1", "foreign1"))); + + // The instance grant covers the managed index, but the foreign index needs the general indices:delete. + verify(outdatedIndexService).delete("managed1"); + verify(outdatedIndexService, never()).delete("foreign1"); + assertThat(response.successfullyPerformed()).isEqualTo(1); + assertThat(response.failures()).extracting(BulkOperationFailure::entityId).containsExactly("foreign1"); + } + + @Test + @WithAuthorization(permissions = {"indices:read", "indices:delete"}) + void bulkDeleteOutdatedRejectsEmptyRequest() { + Assertions.assertThatThrownBy(() -> outdatedIndexResource.bulkDeleteOutdated(new BulkOperationRequest(List.of()))) + .isInstanceOf(jakarta.ws.rs.BadRequestException.class); + } + } diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx index 081642fcd6d7..b1c09715e9c0 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx @@ -16,6 +16,8 @@ */ import React, { useMemo, useState } from 'react'; +import { SystemIndexerIndices } from '@graylog/server-api'; + import { MenuItem } from 'components/bootstrap'; import BulkActionsDropdown from 'components/common/EntityDataTable/BulkActionsDropdown'; import useSelectedEntities from 'components/common/EntityDataTable/hooks/useSelectedEntities'; @@ -30,7 +32,7 @@ import { getBulkIndexActionNotification, runBulkIndexAction, } from './bulkIndexActions'; -import type { BulkIndexActionCandidate } from './bulkIndexActions'; +import type { BulkIndexActionCandidate, BulkIndexActionNotification } from './bulkIndexActions'; import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; import type { PendingIndexStatus } from './hooks/usePendingIncompatibleIndexActions'; @@ -42,6 +44,54 @@ type Props = { refetch: () => void; }; +const showNotification = ({ type, message, title }: BulkIndexActionNotification) => { + if (type === 'success') { + UserNotification.success(message); + } else if (type === 'warning') { + UserNotification.warning(message, title); + } else { + UserNotification.error(message, title); + } +}; + +// Deletes are handled by a single server-side bulk endpoint. Returns the names of the indices that were deleted. +const bulkDeleteIndices = async (bulkAction: BulkIndexActionCandidate): Promise> => { + const entityIds = bulkAction.targetIndices.map((index) => index.index_name); + const { failures } = await SystemIndexerIndices.bulkDeleteOutdated({ entity_ids: entityIds }); + const failedIds = (failures ?? []).map(({ entity_id }) => entity_id); + const succeeded = entityIds.length - failedIds.length; + + if (failedIds.length === 0) { + showNotification({ type: 'success', message: `${succeeded} ${succeeded === 1 ? 'index was' : 'indices were'} deleted.` }); + } else { + const details = (failures ?? []) + .slice(0, 3) + .map(({ entity_id, failure_explanation }) => `${entity_id}: ${failure_explanation}`) + .join('\n'); + const more = failedIds.length > 3 ? `\n...and ${failedIds.length - 3} more.` : ''; + const message = + succeeded > 0 + ? `${succeeded} succeeded, ${failedIds.length} failed.\n${details}${more}` + : `${failedIds.length} ${failedIds.length === 1 ? 'index' : 'indices'} failed.\n${details}${more}`; + + showNotification( + succeeded > 0 + ? { type: 'warning', message, title: 'Some indices could not be deleted' } + : { type: 'error', message, title: 'Could not delete indices' }, + ); + } + + return entityIds.filter((id) => !failedIds.includes(id)); +}; + +// Reindex has no bulk endpoint, so it runs client-side. Returns the names of the indices that were reindexed. +const bulkReindexIndices = async (bulkAction: BulkIndexActionCandidate): Promise> => { + const result = await runBulkIndexAction({ action: bulkAction.action, indices: bulkAction.targetIndices }); + showNotification(getBulkIndexActionNotification(bulkAction, result)); + + return result.successes.map(({ index }) => index.index_name); +}; + const IncompatibleIndicesBulkActions = ({ indices, canArchive, @@ -77,23 +127,12 @@ const IncompatibleIndicesBulkActions = ({ setIsSubmitting(true); try { - const result = await runBulkIndexAction({ - action: confirmedBulkAction.action, - indices: confirmedBulkAction.targetIndices, - }); - - const notification = getBulkIndexActionNotification(confirmedBulkAction, result); - - if (notification.type === 'success') { - UserNotification.success(notification.message); - } else if (notification.type === 'warning') { - UserNotification.warning(notification.message, notification.title); - } else { - UserNotification.error(notification.message, notification.title); - } - - const succeededIds = new Set(result.successes.map(({ index }) => index.index_name)); - setSelectedEntities(selectedEntities.filter((id) => !succeededIds.has(id))); + const succeededIds = + confirmedBulkAction.action === 'delete' + ? await bulkDeleteIndices(confirmedBulkAction) + : await bulkReindexIndices(confirmedBulkAction); + + setSelectedEntities(selectedEntities.filter((id) => !succeededIds.includes(id))); refetch(); setConfirmedBulkAction(undefined); } finally { From d23ae810f8c25c15cf1d66466103ecfe1104c0c6 Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld Date: Fri, 17 Jul 2026 13:30:56 +0200 Subject: [PATCH 7/9] fix test --- .../org/graylog2/indexer/indices/OutdatedIndexServiceIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java b/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java index a29ef64696ef..d710cd8f2ab6 100644 --- a/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java +++ b/full-backend-tests/src/test/java/org/graylog2/indexer/indices/OutdatedIndexServiceIT.java @@ -43,7 +43,7 @@ public class OutdatedIndexServiceIT extends SearchServerBaseTest { public void setUp() throws Exception { indicesAdapter = searchServer().adapters().indicesAdapter(); countsAdapter = searchServer().adapters().countsAdapter(); - outdatedIndexService = new OutdatedIndexService(indicesAdapter, null, null, null); + outdatedIndexService = new OutdatedIndexService(indicesAdapter, null, null, null, null); } @FullBackendTest From a8b605198d8c77ff5f217d95ff5d671877bf46e3 Mon Sep 17 00:00:00 2001 From: Mohamed Ould Hocine Date: Fri, 17 Jul 2026 16:29:17 +0200 Subject: [PATCH 8/9] cleanup + ux improvements --- .../IncompatibleIndexTableActions.tsx | 3 +- .../IncompatibleIndicesBulkActions.test.tsx | 141 ++++++++++++++++++ .../IncompatibleIndicesBulkActions.tsx | 30 ++-- .../IncompatibleIndicesColumnRenderers.tsx | 8 +- .../IncompatibleIndicesTable.test.tsx | 46 +++++- .../IncompatibleIndicesTable.tsx | 12 +- .../OpenSearchUpgradeSection.tsx | 10 +- .../fetchIncompatibleIndices.ts | 9 +- .../incompatibleIndexActions.tsx | 8 +- .../indices/hooks/useIncompatibleIndices.ts | 4 - 10 files changed, 238 insertions(+), 33 deletions(-) create mode 100644 graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.test.tsx diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx index 333fe9240b3c..3369840ead4c 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndexTableActions.tsx @@ -57,7 +57,7 @@ const IncompatibleIndexTableActions = ({ pendingStatus, isArchived, addArchiveDeleteAction, - refetchClusterJobs, + refetchClusterJobs = undefined, refetch, }: Props) => { const actionDefinitions = useIncompatibleIndexActionDefinitions(); @@ -67,7 +67,6 @@ const IncompatibleIndexTableActions = ({ const [isSubmitting, setIsSubmitting] = useState(false); if (pendingStatus?.state === 'archiving') { - // No empty 0% bar flash for jobs that finish almost instantly. return pendingStatus.percent > 0 ? ( . + */ +import * as React from 'react'; +import { render, screen, waitFor } from 'wrappedTestingLibrary'; +import userEvent from '@testing-library/user-event'; + +import { SystemIndexerIndices } from '@graylog/server-api'; + +import asMock from 'helpers/mocking/AsMock'; +import useSelectedEntities from 'components/common/EntityDataTable/hooks/useSelectedEntities'; +import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; +import UserNotification from 'util/UserNotification'; + +import IncompatibleIndicesBulkActions from './IncompatibleIndicesBulkActions'; +import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; + +jest.mock('@graylog/server-api', () => ({ + SystemIndexerIndices: { + bulkDeleteOutdated: jest.fn(), + }, +})); +jest.mock('components/common/EntityDataTable/hooks/useSelectedEntities'); +jest.mock('logic/telemetry/useSendTelemetry'); +jest.mock('util/UserNotification', () => ({ success: jest.fn(), warning: jest.fn(), error: jest.fn() })); + +const makeIndex = (indexName: string): IncompatibleIndexRow => ({ + id: indexName, + index_name: indexName, + version: '7.10.2', + warm_index: false, + managed_index: false, + system_index: false, + active_write_index: null, + begin: null, + end: null, +}); + +const indices = [makeIndex('legacy_0'), makeIndex('legacy_1')]; + +describe('IncompatibleIndicesBulkActions', () => { + const setSelectedEntities = jest.fn(); + const refetch = jest.fn(); + + const renderBulkActions = () => + render( + , + ); + + const confirmBulkDelete = async () => { + await userEvent.click(screen.getByRole('button', { name: /bulk actions/i })); + await userEvent.click(await screen.findByRole('menuitem', { name: /delete all \(2\)/i })); + await userEvent.click(await screen.findByRole('button', { name: /^delete all$/i })); + }; + + beforeEach(() => { + jest.clearAllMocks(); + asMock(useSelectedEntities).mockReturnValue({ + selectedEntities: indices.map(({ id }) => id), + setSelectedEntities, + selectEntity: jest.fn(), + deselectEntity: jest.fn(), + toggleEntitySelect: jest.fn(), + isSomeRowsSelected: false, + isAllRowsSelected: true, + }); + asMock(useSendTelemetry).mockReturnValue(jest.fn()); + }); + + it('deletes all selected indices, clears the selection, and refreshes the table', async () => { + asMock(SystemIndexerIndices.bulkDeleteOutdated).mockResolvedValue({ + successfully_performed: 2, + failures: [], + errors: [], + }); + renderBulkActions(); + + await confirmBulkDelete(); + + await waitFor(() => { + expect(SystemIndexerIndices.bulkDeleteOutdated).toHaveBeenCalledWith({ + entity_ids: ['legacy_0', 'legacy_1'], + }); + expect(UserNotification.success).toHaveBeenCalledWith('2 indices were deleted.'); + expect(setSelectedEntities).toHaveBeenCalledWith([]); + expect(refetch).toHaveBeenCalledTimes(1); + }); + }); + + it('keeps failed indices selected and warns with the failure details', async () => { + asMock(SystemIndexerIndices.bulkDeleteOutdated).mockResolvedValue({ + successfully_performed: 1, + failures: [{ entity_id: 'legacy_1', failure_explanation: 'Delete failed' }], + errors: [], + }); + renderBulkActions(); + + await confirmBulkDelete(); + + await waitFor(() => { + expect(UserNotification.warning).toHaveBeenCalledWith( + '1 succeeded, 1 failed.\nlegacy_1: Delete failed', + 'Some indices could not be deleted', + ); + expect(setSelectedEntities).toHaveBeenCalledWith(['legacy_1']); + expect(refetch).toHaveBeenCalledTimes(1); + }); + }); + + it('reports request failures without changing the selection or refreshing', async () => { + asMock(SystemIndexerIndices.bulkDeleteOutdated).mockRejectedValue(new Error('Backend unavailable')); + renderBulkActions(); + + await confirmBulkDelete(); + + await waitFor(() => + expect(UserNotification.error).toHaveBeenCalledWith('Backend unavailable', 'Could not delete all.'), + ); + expect(setSelectedEntities).not.toHaveBeenCalled(); + expect(refetch).not.toHaveBeenCalled(); + }); +}); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx index b1c09715e9c0..079879a592ce 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx @@ -22,16 +22,13 @@ import { MenuItem } from 'components/bootstrap'; import BulkActionsDropdown from 'components/common/EntityDataTable/BulkActionsDropdown'; import useSelectedEntities from 'components/common/EntityDataTable/hooks/useSelectedEntities'; import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; +import extractErrorMessage from 'util/extractErrorMessage'; import UserNotification from 'util/UserNotification'; import { TELEMETRY_DEFAULTS } from './telemetry'; import BulkIndexActionConfirmDialog from './BulkIndexActionConfirmDialog'; import { CORE_ACTION_DEFINITIONS } from './incompatibleIndexActions'; -import { - getBulkIndexActionCandidates, - getBulkIndexActionNotification, - runBulkIndexAction, -} from './bulkIndexActions'; +import { getBulkIndexActionCandidates, getBulkIndexActionNotification, runBulkIndexAction } from './bulkIndexActions'; import type { BulkIndexActionCandidate, BulkIndexActionNotification } from './bulkIndexActions'; import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; import type { PendingIndexStatus } from './hooks/usePendingIncompatibleIndexActions'; @@ -54,7 +51,6 @@ const showNotification = ({ type, message, title }: BulkIndexActionNotification) } }; -// Deletes are handled by a single server-side bulk endpoint. Returns the names of the indices that were deleted. const bulkDeleteIndices = async (bulkAction: BulkIndexActionCandidate): Promise> => { const entityIds = bulkAction.targetIndices.map((index) => index.index_name); const { failures } = await SystemIndexerIndices.bulkDeleteOutdated({ entity_ids: entityIds }); @@ -62,7 +58,10 @@ const bulkDeleteIndices = async (bulkAction: BulkIndexActionCandidate): Promise< const succeeded = entityIds.length - failedIds.length; if (failedIds.length === 0) { - showNotification({ type: 'success', message: `${succeeded} ${succeeded === 1 ? 'index was' : 'indices were'} deleted.` }); + showNotification({ + type: 'success', + message: `${succeeded} ${succeeded === 1 ? 'index was' : 'indices were'} deleted.`, + }); } else { const details = (failures ?? []) .slice(0, 3) @@ -84,7 +83,6 @@ const bulkDeleteIndices = async (bulkAction: BulkIndexActionCandidate): Promise< return entityIds.filter((id) => !failedIds.includes(id)); }; -// Reindex has no bulk endpoint, so it runs client-side. Returns the names of the indices that were reindexed. const bulkReindexIndices = async (bulkAction: BulkIndexActionCandidate): Promise> => { const result = await runBulkIndexAction({ action: bulkAction.action, indices: bulkAction.targetIndices }); showNotification(getBulkIndexActionNotification(bulkAction, result)); @@ -110,10 +108,17 @@ const IncompatibleIndicesBulkActions = ({ ); const candidates = useMemo( - () => getBulkIndexActionCandidates({ indices: selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames }), + () => + getBulkIndexActionCandidates({ indices: selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames }), [selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames], ); + const handleCancel = () => { + if (!isSubmitting) { + setConfirmedBulkAction(undefined); + } + }; + const handleConfirm = async () => { if (!confirmedBulkAction || isSubmitting) { return; @@ -135,6 +140,11 @@ const IncompatibleIndicesBulkActions = ({ setSelectedEntities(selectedEntities.filter((id) => !succeededIds.includes(id))); refetch(); setConfirmedBulkAction(undefined); + } catch (errorThrown) { + UserNotification.error( + extractErrorMessage(errorThrown), + `Could not ${confirmedBulkAction.confirmText.toLowerCase()}.`, + ); } finally { setIsSubmitting(false); } @@ -153,7 +163,7 @@ const IncompatibleIndicesBulkActions = ({ setConfirmedBulkAction(undefined)} + onCancel={handleCancel} onConfirm={handleConfirm} /> )} diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx index 65737575b4a4..7a297753324d 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesColumnRenderers.tsx @@ -15,7 +15,6 @@ * . */ import * as React from 'react'; - import type { ColorVariant } from '@graylog/sawmill'; import { Label } from 'components/bootstrap'; @@ -30,7 +29,6 @@ export const DEFAULT_DISPLAYED_COLUMNS = ['index_name', 'category', 'version', ' type Badge = { text: string; style: ColorVariant }; -// The primary, mutually exclusive classification; matches the backend `category` field precedence. const primaryTypeBadge = (index: IncompatibleIndex): Badge => { if (index.system_index) { return { text: 'System', style: 'info' }; @@ -75,7 +73,11 @@ export const createColumnRenderers = ( index_name: { minWidth: 300, renderCell: (_value, index) => { - const isArchived = isIndexArchived(index.index_name, pendingIndexStatuses.get(index.index_name), archivedIndexNames); + const isArchived = isIndexArchived( + index.index_name, + pendingIndexStatuses.get(index.index_name), + archivedIndexNames, + ); return ( diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx index 04657d66f5a7..c7be0ae56b31 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx @@ -17,9 +17,12 @@ import * as React from 'react'; import { render, screen } from 'wrappedTestingLibrary'; +import { SystemIndexerIndices } from '@graylog/server-api'; + import asMock from 'helpers/mocking/AsMock'; import type { PaginatedEntityTableProps } from 'components/common/PaginatedEntityTable/PaginatedEntityTable'; import useCanArchive from 'components/indices/hooks/useCanArchive'; +import type { SearchParams } from 'stores/PaginationTypes'; import IncompatibleIndicesTable from './IncompatibleIndicesTable'; import { createColumnRenderers } from './IncompatibleIndicesColumnRenderers'; @@ -34,6 +37,9 @@ jest.mock('components/common/PaginatedEntityTable', () => ({ useTableFetchContext: jest.fn(), })); +jest.mock('@graylog/server-api', () => ({ + SystemIndexerIndices: { listOutdatedIndices: jest.fn() }, +})); jest.mock('components/indices/hooks/useCanArchive'); jest.mock('./hooks/useArchivedIndexNames'); jest.mock('./hooks/usePendingIncompatibleIndexActions'); @@ -51,6 +57,14 @@ const makeIndex = (overrides: Partial): IncompatibleIndexR ...overrides, }); +const searchParams: SearchParams = { + page: 2, + pageSize: 20, + query: '', + sort: { attributeId: 'index_name', direction: 'asc' }, + filters: undefined, +}; + describe('IncompatibleIndicesTable', () => { beforeEach(() => { jest.clearAllMocks(); @@ -73,7 +87,10 @@ describe('IncompatibleIndicesTable', () => { expect(screen.getByText('Paginated incompatible indices')).toBeInTheDocument(); expect(mockPaginatedEntityTable).toHaveBeenCalledTimes(1); - const callProps = mockPaginatedEntityTable.mock.calls[0][0] as PaginatedEntityTableProps; + const callProps = mockPaginatedEntityTable.mock.calls[0][0] as PaginatedEntityTableProps< + IncompatibleIndexRow, + unknown + >; expect(callProps.fetchEntities).toBe(fetchIncompatibleIndices); expect(callProps.keyFn).toBe(incompatibleIndicesKeyFn); expect(callProps.entityAttributesAreCamelCase).toBe(false); @@ -84,6 +101,33 @@ describe('IncompatibleIndicesTable', () => { }); }); +describe('fetchIncompatibleIndices', () => { + it('maps the nested pagination total and adds the entity id', async () => { + const index = { + index_name: 'legacy-index', + version: '7.10.2', + warm_index: false, + managed_index: false, + system_index: false, + active_write_index: null, + begin: null, + end: null, + }; + const listOutdatedIndices = asMock(SystemIndexerIndices.listOutdatedIndices).mockResolvedValue({ + elements: [index], + attributes: [], + pagination: { total: 42 }, + total: 0, + } as never); + + const result = await fetchIncompatibleIndices(searchParams); + + expect(listOutdatedIndices).toHaveBeenCalledWith('index_name', 2, 20, '', 'asc'); + expect(result.list).toEqual([{ ...index, id: 'legacy-index' }]); + expect(result.pagination).toEqual({ total: 42 }); + }); +}); + describe('createColumnRenderers', () => { const renderers = createColumnRenderers(new Map(), new Set()); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx index c2bb793acff4..74aad3cdd53f 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx @@ -44,7 +44,7 @@ const TABLE_LAYOUT = { entityTableId: 'incompatible_indices', defaultSort: { attributeId: 'index_name', direction: 'asc' as const }, defaultDisplayedAttributes: DEFAULT_DISPLAYED_COLUMNS, - defaultPageSize: 20, + defaultPageSize: 10, defaultColumnOrder: ['index_name', 'category', 'version', 'begin', 'end'], }; @@ -70,7 +70,6 @@ const IncompatibleIndicesTable = () => { canArchive, }); - // Suppress archive actions while an archive job runs — the backend archive job is single-concurrency. const archiveActionsAvailable = canArchive && !isArchiveJobRunning; const handleDataLoaded = useCallback((data: IncompatibleIndicesResponse) => { @@ -102,7 +101,14 @@ const IncompatibleIndicesTable = () => { /> ); }, - [pendingIndexStatuses, archivedIndexNames, archiveActionsAvailable, addArchiveDeleteAction, refetchClusterJobs, refetch], + [ + pendingIndexStatuses, + archivedIndexNames, + archiveActionsAvailable, + addArchiveDeleteAction, + refetchClusterJobs, + refetch, + ], ); const bulkSelection = useMemo( diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/OpenSearchUpgradeSection.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/OpenSearchUpgradeSection.tsx index c30cb79a9c2b..9c1eb3fbd44b 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/OpenSearchUpgradeSection.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/OpenSearchUpgradeSection.tsx @@ -49,6 +49,12 @@ const DisabledHint = styled.p( `, ); +const ActionsRow = styled(Row)( + ({ theme }) => css` + margin-top: ${theme.spacings.md}; + `, +); + const MIN_NODES_FOR_ROLLING_UPGRADE = 3; const TELEMETRY_DEFAULTS = { app_pathname: 'datanode', app_section: 'opensearch-upgrade' } as const; @@ -169,7 +175,7 @@ const OpenSearchUpgradeSection = () => { )} - + {showStartAction && ( @@ -197,7 +203,7 @@ const OpenSearchUpgradeSection = () => { Data Nodes' embedded OpenSearch is already up to date. )} - + {showRollingUpgradeStatus && ( diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts index 563e5347ea03..64868e7dade2 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/fetchIncompatibleIndices.ts @@ -20,7 +20,6 @@ import FiltersForQueryParams from 'components/common/EntityFilters/FiltersForQue import type { Attribute, SearchParams } from 'stores/PaginationTypes'; import type { IncompatibleIndex } from 'components/indices/hooks/useIncompatibleIndices'; -// Entity tables require an `id` on every row; the index name is a stable natural key. export type IncompatibleIndexRow = IncompatibleIndex & { id: string }; export type IncompatibleIndicesResponse = { @@ -35,16 +34,14 @@ type ListOutdatedIndicesSort = Parameters[4]; export const fetchIncompatibleIndices = (searchParams: SearchParams): Promise => { - // This endpoint is backed by an in-memory Lucene search that only understands the `query` string - // (there is no separate `filters` param), so fold the active filter chips into the query. - const query = [searchParams.query, ...FiltersForQueryParams(searchParams.filters)].filter(Boolean).join(' '); + const query = [searchParams.query, ...(FiltersForQueryParams(searchParams.filters) ?? [])].filter(Boolean).join(' '); const sort = (searchParams.sort?.attributeId ?? 'index_name') as ListOutdatedIndicesSort; const order = (searchParams.sort?.direction ?? 'asc') as ListOutdatedIndicesOrder; return SystemIndexerIndices.listOutdatedIndices(sort, searchParams.page, searchParams.pageSize, query, order).then( - ({ elements, attributes, total }) => ({ + ({ elements, attributes, pagination }) => ({ list: elements.map((element) => ({ ...element, id: element.index_name })) as Array, - pagination: { total }, + pagination: { total: pagination.total }, attributes, }), ); diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx index e8a2a553276b..9e3f452e6249 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/incompatibleIndexActions.tsx @@ -97,7 +97,9 @@ export const CORE_ACTION_DEFINITIONS: Record{index.index_name} is the active write index of its index set and still receives new messages. Rotating starts a new write index on the current OpenSearch version.

-

Afterwards, {index.index_name} can be archived or deleted.

+

+ Afterwards, {index.index_name} can be archived or deleted. +

), run: rotateWriteIndex, @@ -118,7 +120,9 @@ const archiveDeleteDefinition = (archive: IndexArchiveBinding | undefined): Acti

), run: (index) => - archive ? archive.archiveAndDeleteIndex(index.index_name) : Promise.reject(new Error('Archiving is not available.')), + archive + ? archive.archiveAndDeleteIndex(index.index_name) + : Promise.reject(new Error('Archiving is not available.')), successMessage: (index) => `Archive and delete job for "${index.index_name}" was started.`, telemetryEventType: TELEMETRY_EVENT_TYPE.DATANODE_OPENSEARCH_UPGRADE.INDEX_ARCHIVE_AND_DELETE_CONFIRMED, getPendingArchiveTracking: (index, response) => ({ diff --git a/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts b/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts index 05f1e2a81c88..9874d97f3020 100644 --- a/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts +++ b/graylog2-web-interface/src/components/indices/hooks/useIncompatibleIndices.ts @@ -24,11 +24,8 @@ export type IncompatibleIndex = { warm_index: boolean; managed_index: boolean; system_index: boolean; - /** Id of the index set this index is the active write index of, `null` for all other indices. */ active_write_index: string | null; - /** Start of the message time range stored in the index, `null` when unknown / not calculated. */ begin?: string | null; - /** End of the message time range stored in the index, `null` when unknown / not calculated. */ end?: string | null; }; @@ -42,7 +39,6 @@ const useIncompatibleIndices = () => { refetch, } = useQuery({ queryKey: ['incompatibleIndices'], - // No error toast: the panels render a persistent error state, and background retries would spam toasts. queryFn: () => SystemIndexerIndices.getOutdatedIndices() as Promise>, retry: 2, refetchInterval: (query) => (query.state.status === 'error' ? ERROR_REFETCH_INTERVAL_MS : false), From eaac834a7a70edfb3363225e14b78046aeafe025 Mon Sep 17 00:00:00 2001 From: Mohamed Ould Hocine Date: Mon, 20 Jul 2026 16:40:18 +0200 Subject: [PATCH 9/9] bulk action for PaginatedEntityTable --- .../IncompatibleIndicesBulkActions.tsx | 19 ++-- .../IncompatibleIndicesTable.test.tsx | 73 +++++++++++- .../IncompatibleIndicesTable.tsx | 105 +++++++++--------- 3 files changed, 130 insertions(+), 67 deletions(-) diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx index 079879a592ce..70f66ce36bd4 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesBulkActions.tsx @@ -14,7 +14,7 @@ * along with this program. If not, see * . */ -import React, { useMemo, useState } from 'react'; +import React, { useState } from 'react'; import { SystemIndexerIndices } from '@graylog/server-api'; @@ -102,16 +102,13 @@ const IncompatibleIndicesBulkActions = ({ const [confirmedBulkAction, setConfirmedBulkAction] = useState(); const [isSubmitting, setIsSubmitting] = useState(false); - const selectedIndices = useMemo( - () => indices.filter((index) => selectedEntities.includes(index.id)), - [indices, selectedEntities], - ); - - const candidates = useMemo( - () => - getBulkIndexActionCandidates({ indices: selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames }), - [selectedIndices, canArchive, pendingIndexStatuses, archivedIndexNames], - ); + const selectedIndices = indices.filter((index) => selectedEntities.includes(index.id)); + const candidates = getBulkIndexActionCandidates({ + indices: selectedIndices, + canArchive, + pendingIndexStatuses, + archivedIndexNames, + }); const handleCancel = () => { if (!isSubmitting) { diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx index c7be0ae56b31..28bae56b3fa0 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.test.tsx @@ -16,6 +16,7 @@ */ import * as React from 'react'; import { render, screen } from 'wrappedTestingLibrary'; +import { act } from 'react'; import { SystemIndexerIndices } from '@graylog/server-api'; @@ -27,7 +28,7 @@ import type { SearchParams } from 'stores/PaginationTypes'; import IncompatibleIndicesTable from './IncompatibleIndicesTable'; import { createColumnRenderers } from './IncompatibleIndicesColumnRenderers'; import { fetchIncompatibleIndices, incompatibleIndicesKeyFn } from './fetchIncompatibleIndices'; -import type { IncompatibleIndexRow } from './fetchIncompatibleIndices'; +import type { IncompatibleIndexRow, IncompatibleIndicesResponse } from './fetchIncompatibleIndices'; import useArchivedIndexNames from './hooks/useArchivedIndexNames'; import usePendingIncompatibleIndexActions from './hooks/usePendingIncompatibleIndexActions'; @@ -65,6 +66,17 @@ const searchParams: SearchParams = { filters: undefined, }; +const makeResponse = (list: Array): IncompatibleIndicesResponse => ({ + list, + pagination: { total: list.length }, + attributes: [], +}); + +const latestTableProps = ( + mockPaginatedEntityTable: jest.Mock, +): PaginatedEntityTableProps => + mockPaginatedEntityTable.mock.calls[mockPaginatedEntityTable.mock.calls.length - 1][0]; + describe('IncompatibleIndicesTable', () => { beforeEach(() => { jest.clearAllMocks(); @@ -99,6 +111,65 @@ describe('IncompatibleIndicesTable', () => { expect(callProps.bulkSelection.actions).toBeTruthy(); expect(callProps.columnRenderers.attributes).toHaveProperty('index_name'); }); + + it('retains selected index data across pages and removes deselected indices', () => { + const { default: PaginatedEntityTable } = jest.requireMock('components/common/PaginatedEntityTable'); + const mockPaginatedEntityTable = asMock(PaginatedEntityTable); + const firstPageIndex = makeIndex({ id: 'legacy_0', index_name: 'legacy_0' }); + const secondPageIndex = makeIndex({ id: 'legacy_1', index_name: 'legacy_1' }); + + render(); + + act(() => { + latestTableProps(mockPaginatedEntityTable).onDataLoaded?.(makeResponse([firstPageIndex])); + }); + + act(() => { + latestTableProps(mockPaginatedEntityTable).bulkSelection.onChangeSelection?.( + [firstPageIndex.id], + [firstPageIndex], + ); + }); + + act(() => { + latestTableProps(mockPaginatedEntityTable).onDataLoaded?.(makeResponse([secondPageIndex])); + }); + + act(() => { + latestTableProps(mockPaginatedEntityTable).bulkSelection.onChangeSelection?.( + [firstPageIndex.id, secondPageIndex.id], + [secondPageIndex], + ); + }); + + const bulkActions = latestTableProps(mockPaginatedEntityTable).bulkSelection.actions as React.ReactElement<{ + indices: Array; + }>; + + expect(bulkActions.props.indices.map(({ id }) => id)).toEqual(['legacy_0', 'legacy_1']); + expect(useArchivedIndexNames).toHaveBeenLastCalledWith(['legacy_0', 'legacy_1'], true); + expect(usePendingIncompatibleIndexActions).toHaveBeenLastCalledWith( + expect.objectContaining({ incompatibleIndices: [firstPageIndex, secondPageIndex] }), + ); + + act(() => { + latestTableProps(mockPaginatedEntityTable).bulkSelection.onChangeSelection?.( + [secondPageIndex.id], + [secondPageIndex], + ); + }); + + const actionsAfterDeselection = latestTableProps(mockPaginatedEntityTable).bulkSelection + .actions as React.ReactElement<{ + indices: Array; + }>; + + expect(actionsAfterDeselection.props.indices).toEqual([secondPageIndex]); + expect(useArchivedIndexNames).toHaveBeenLastCalledWith(['legacy_1'], true); + expect(usePendingIncompatibleIndexActions).toHaveBeenLastCalledWith( + expect.objectContaining({ incompatibleIndices: [secondPageIndex] }), + ); + }); }); describe('fetchIncompatibleIndices', () => { diff --git a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx index 74aad3cdd53f..8cd1fed4d815 100644 --- a/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx +++ b/graylog2-web-interface/src/components/datanode/opensearch-upgrade/IncompatibleIndicesTable.tsx @@ -14,9 +14,11 @@ * along with this program. If not, see * . */ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useState } from 'react'; import styled, { css } from 'styled-components'; import { useQueryClient } from '@tanstack/react-query'; +import keyBy from 'lodash/keyBy'; +import pickBy from 'lodash/pickBy'; import { PaginatedEntityTable } from 'components/common'; import useCanArchive from 'components/indices/hooks/useCanArchive'; @@ -52,18 +54,17 @@ const IncompatibleIndicesTable = () => { const queryClient = useQueryClient(); const canArchive = useCanArchive(); const [loadedIndices, setLoadedIndices] = useState>([]); + const [selectedIndicesData, setSelectedIndicesData] = useState>({}); const [hasLoaded, setHasLoaded] = useState(false); - const refetch = useCallback( - () => queryClient.invalidateQueries({ queryKey: INCOMPATIBLE_INDICES_QUERY_KEY }), - [queryClient], - ); - - const incompatibleIndexNames = useMemo(() => loadedIndices.map((index) => index.index_name), [loadedIndices]); + const refetch = () => queryClient.invalidateQueries({ queryKey: INCOMPATIBLE_INDICES_QUERY_KEY }); + const selectedIndices = Object.values(selectedIndicesData); + const trackedIndices = Object.values({ ...selectedIndicesData, ...keyBy(loadedIndices, 'id') }); + const incompatibleIndexNames = trackedIndices.map((index) => index.index_name); const archivedIndexNames = useArchivedIndexNames(incompatibleIndexNames, canArchive); const { pendingIndexStatuses, addArchiveDeleteAction, isArchiveJobRunning, refetchClusterJobs } = usePendingIncompatibleIndexActions({ - incompatibleIndices: loadedIndices, + incompatibleIndices: trackedIndices, isLoading: !hasLoaded, isError: false, refetch, @@ -72,59 +73,53 @@ const IncompatibleIndicesTable = () => { const archiveActionsAvailable = canArchive && !isArchiveJobRunning; - const handleDataLoaded = useCallback((data: IncompatibleIndicesResponse) => { + const handleDataLoaded = (data: IncompatibleIndicesResponse) => { setLoadedIndices(data.list); setHasLoaded(true); - }, []); + }; - const columnRenderers = useMemo( - () => createColumnRenderers(pendingIndexStatuses, archivedIndexNames), - [pendingIndexStatuses, archivedIndexNames], - ); + const columnRenderers = createColumnRenderers(pendingIndexStatuses, archivedIndexNames); - const renderActions = useCallback( - (index: IncompatibleIndexRow) => { - const pendingStatus = pendingIndexStatuses.get(index.index_name); - const isArchived = - pendingStatus?.state !== 'archiving' && - (archivedIndexNames.has(index.index_name) || pendingStatus?.state === 'archived'); + const renderActions = (index: IncompatibleIndexRow) => { + const pendingStatus = pendingIndexStatuses.get(index.index_name); + const isArchived = + pendingStatus?.state !== 'archiving' && + (archivedIndexNames.has(index.index_name) || pendingStatus?.state === 'archived'); - return ( - - ); - }, - [ - pendingIndexStatuses, - archivedIndexNames, - archiveActionsAvailable, - addArchiveDeleteAction, - refetchClusterJobs, - refetch, - ], - ); + return ( + + ); + }; - const bulkSelection = useMemo( - () => ({ - actions: ( - - ), - }), - [loadedIndices, archiveActionsAvailable, pendingIndexStatuses, archivedIndexNames, refetch], - ); + const bulkSelection = { + onChangeSelection: (selectedItemsIds: Array, list: Readonly>) => { + setSelectedIndicesData((cur) => { + const selectedItemsIdsSet = new Set(selectedItemsIds); + const selectedCurrentItems = pickBy(cur, (_, indexId) => selectedItemsIdsSet.has(indexId)); + const selectedCurrentEntries = list.filter(({ id }) => selectedItemsIdsSet.has(id)); + const currentEntriesById = keyBy(selectedCurrentEntries, 'id'); + + return { ...selectedCurrentItems, ...currentEntriesById }; + }); + }, + actions: ( + + ), + }; return ( <>