From d3bb39b92ff3fe55fdce7d3f2cf96ac0dcf3989f Mon Sep 17 00:00:00 2001 From: Sagar Darji <42430138+darjisagar7@users.noreply.github.com> Date: Sun, 7 Jun 2026 00:10:32 +0530 Subject: [PATCH 01/11] [Pluggable Dataformat] Adding support for FieldTypeCapabilities (#21733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR introduces a capability-based routing system for field types in the Pluggable Dataformat architecture. Instead of each field mapper implementing a parseCreateFieldForPluggableFormat method (duplicating parsing logic per format), field types now declare their capabilities (e.g., POINT_RANGE, COLUMNAR_STORAGE, FULL_TEXT_SEARCH), and the data format infrastructure uses these declarations to determine which format handles which aspect of a field. Key changes: 1. Capability declaration — MappedFieldType gains a searchCapability() method. Each field mapper returns its appropriate capability (e.g., numeric types return POINT_RANGE, keyword returns FULL_TEXT_SEARCH). 2. FieldCapabilityAssigner — A new component that walks configured data formats in priority order and assigns capabilities to field types greedily (primary format claims first, secondary fills gaps). 3. Removed redundant pluggable format overrides — Mappers like SearchAsYouTypeFieldMapper, TokenCountFieldMapper, RankFeatureFieldMapper no longer need custom parseCreateFieldForPluggableFormat implementations — the capability system handles routing automatically. 4. Test infrastructure — New test utilities (DataFormatTestUtils, MockParquetDataFormatPlugin) for testing capability assignment and pluggable format behavior. The net effect: data format plugins (parquet, lucene) declare what capabilities they support per field type, and the system automatically routes each field's data to the appropriate format — without field mappers needing to know about specific data formats. --------- Signed-off-by: Sagar Darji Signed-off-by: Mohit Godwani Co-authored-by: Sagar Darji Co-authored-by: Mohit Godwani --- .../index/mapper/ScaledFloatFieldMapper.java | 6 + .../mapper/SearchAsYouTypeFieldMapper.java | 5 - .../index/mapper/TokenCountFieldMapper.java | 9 - .../mapper/RankFeatureFieldMapperTests.java | 14 - .../mapper/RankFeaturesFieldMapperTests.java | 14 - .../SearchAsYouTypeFieldMapperTests.java | 114 --- .../mapper/TokenCountFieldMapperTests.java | 83 +- .../mapper/ParentJoinFieldMapperTests.java | 130 +-- .../ICUCollationKeywordFieldMapper.java | 9 - .../ICUCollationKeywordFieldMapperTests.java | 59 -- .../AnnotatedTextFieldMapperTests.java | 83 -- .../AnnotatedTextFieldMapper.java | 9 - .../murmur3/Murmur3FieldMapperTests.java | 75 -- .../be/datafusion/BaseScalarFunctionIT.java | 2 +- .../be/datafusion/CoordinatorJoinIT.java | 2 +- .../CoordinatorJoinMultiNodeIT.java | 2 +- .../be/lucene/LuceneDataFormat.java | 41 +- .../be/lucene/LuceneFieldFactoryRegistry.java | 34 +- .../opensearch/be/lucene/LucenePlugin.java | 2 +- .../be/lucene/index/LuceneDocumentInput.java | 44 +- .../be/lucene/index/DeleterImplTests.java | 820 ++++++++++++++++ .../index/LuceneDocumentInputTests.java | 42 +- .../LuceneIndexingExecutionEngineTests.java | 23 +- .../lucene/index/LucenePluginBaseTests.java | 58 ++ .../index/LuceneWriterSortedFlushTests.java | 8 +- .../be/lucene/index/LuceneWriterTests.java | 57 +- .../composite/AbstractCompositeEngineIT.java | 7 +- .../composite/AbstractSortedRefreshIT.java | 538 +++++++++++ .../composite/CompositeCommitDeletionIT.java | 6 +- .../CompositeConcurrentIndexingIT.java | 2 +- .../composite/CompositeDataCoherenceIT.java | 546 +++++++++++ .../composite/CompositeDynamicMappingIT.java | 146 +-- .../composite/CompositeFieldCapabilityIT.java | 894 ++++++++++++++++++ .../CompositeFieldConfigRandomizedIT.java | 345 +++++++ .../composite/CompositeMergeIT.java | 47 +- .../CompositeParquet3TierSettingsIT.java | 10 +- .../composite/CompositeParquetIndexIT.java | 526 +---------- .../CompositeParquetRowGroupSettingsIT.java | 2 +- .../CompositeParquetSettingsValidationIT.java | 22 +- .../composite/CompositeRefreshSortedIT.java | 651 +------------ .../CompositeRefreshSortedParquetOnlyIT.java | 142 +++ .../DataFormatAwareMultiIndexRecoveryIT.java | 2 +- ...atAwareMultiIndexRecoveryWithLuceneIT.java | 40 - ...taFormatAwarePeerRecoveryWithLuceneIT.java | 40 - .../DataFormatAwareReadonlyEngineBaseIT.java | 6 +- .../DataFormatAwareRemoteStoreRecoveryIT.java | 4 +- ...tAwareRemoteStoreRecoveryWithLuceneIT.java | 42 - .../DataFormatAwareReplicationBaseIT.java | 4 +- .../DataFormatAwareReplicationIT.java | 4 +- ...AwareReplicationPromotionWithLuceneIT.java | 50 - ...ataFormatAwareReplicationWithLuceneIT.java | 40 - ...FormatAwareRestoreShallowSnapshotV2IT.java | 4 +- ...eRestoreShallowSnapshotV2WithLuceneIT.java | 47 - .../composite/DataFormatAwareUploadIT.java | 2 +- .../FailableParquetDataFormatPlugin.java | 4 +- .../ParquetOnlyDataFormatPlugin.java | 74 ++ .../composite/CompositeDataFormatPlugin.java | 95 ++ .../CompositeAssignCapabilitiesTests.java | 352 +++++++ .../benchmark/VSRRotationBenchmark.java | 9 + .../parquet/engine/ParquetDataFormat.java | 13 +- .../parquet/fields/ArrowFieldRegistry.java | 18 +- .../parquet/fields/ArrowSchemaBuilder.java | 19 +- .../parquet/fields/ParquetField.java | 13 + .../fields/core/data/BinaryParquetField.java | 8 + .../fields/core/data/BooleanParquetField.java | 12 + .../core/data/date/DateNanosParquetField.java | 12 + .../core/data/date/DateParquetField.java | 12 + .../core/data/number/ByteParquetField.java | 3 +- .../core/data/number/DoubleParquetField.java | 3 +- .../core/data/number/FloatParquetField.java | 3 +- .../data/number/HalfFloatParquetField.java | 3 +- .../core/data/number/IntegerParquetField.java | 3 +- .../core/data/number/LongParquetField.java | 3 +- .../core/data/number/NumericParquetField.java | 29 + .../core/data/number/ShortParquetField.java | 3 +- .../data/number/TokenCountParquetField.java | 3 +- .../data/number/UnsignedLongParquetField.java | 3 +- .../fields/core/data/text/IpParquetField.java | 11 + .../core/data/text/TextParquetField.java | 11 + .../fields/core/metadata/IdParquetField.java | 12 + .../core/metadata/IgnoredParquetField.java | 10 + .../core/metadata/IndexParquetField.java | 37 + .../core/metadata/RoutingParquetField.java | 11 + .../fields/plugins/MetadataFieldPlugin.java | 16 + .../parquet/writer/ParquetDocumentInput.java | 17 +- .../opensearch/parquet/ParquetBaseTests.java | 117 +++ .../ParquetDataFormatAwareEngineTests.java | 145 ++- .../engine/ParquetDataFormatTests.java | 5 +- .../engine/ParquetIndexingEngineTests.java | 39 +- .../plugins/MetadataFieldPluginTests.java | 6 +- .../parquet/vsr/VSRManagerTests.java | 24 +- .../writer/ParquetDocumentInputTests.java | 29 +- .../parquet/writer/ParquetWriterTests.java | 11 +- .../AnalyticsQueryTaskCleanupIT.java | 4 +- .../ReduceThreadPoolCleanupIT.java | 4 +- .../cancellation/SearchCancellationIT.java | 4 +- .../resilience/CoordinatorResilienceIT.java | 4 +- .../CoordinatorTopologyTestBase.java | 4 +- .../CoordinatorTransportStressIT.java | 4 +- .../resilience/MaxShardsPerQueryIT.java | 4 +- .../analytics/resilience/MemoryGuardIT.java | 6 +- .../resilience/ReduceEarlyTerminationIT.java | 8 +- .../resilience/ReduceTargetPartitionsIT.java | 4 +- .../analytics/resilience/ShardFailoverIT.java | 4 +- .../SpillDisabledParallelismIT.java | 4 +- .../sql/AnalyticsSearchSlowLogIT.java | 4 +- .../opensearch/analytics/sql/ValuesSqlIT.java | 6 +- .../opensearch/analytics/sql/WindowSqlIT.java | 4 +- .../datafusion/QtfDerivedAboveProjectIT.java | 6 +- .../be/datafusion/QtfSubstraitDumpIT.java | 10 +- .../parquet/ParquetOnlyDataFormatPlugin.java | 73 ++ .../org/opensearch/analytics/qa/AliasIT.java | 1 + .../analytics/qa/CoordinatorReduceIT.java | 10 +- .../qa/CoordinatorReduceMemtableIT.java | 2 +- .../opensearch/analytics/qa/DataStreamIT.java | 3 + .../analytics/qa/FieldTypeCoverageIT.java | 4 +- .../analytics/qa/IndexPatternUnionIT.java | 1 + .../qa/ListAggregateMultiTypeIT.java | 2 +- .../analytics/qa/MultiIndexQueryShapesIT.java | 2 +- .../analytics/qa/ObjectFieldIT.java | 1 + .../analytics/qa/ParquetDataFusionIT.java | 3 +- .../qa/StreamingCoordinatorReduceIT.java | 2 +- .../extensive_coverage/mapping_test_data.json | 1 - .../java/org/opensearch/get/GetActionIT.java | 10 +- .../org/opensearch/index/IndexService.java | 3 +- .../org/opensearch/index/IndexSettings.java | 2 +- .../engine/dataformat/DataFormatPlugin.java | 74 ++ .../engine/dataformat/DataFormatRegistry.java | 28 + .../dataformat/FieldTypeCapabilities.java | 4 +- .../engine/exec/PrimaryTermFieldType.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 27 +- .../index/mapper/BooleanFieldMapper.java | 6 + .../mapper/ConstantKeywordFieldMapper.java | 6 + .../index/mapper/DateFieldMapper.java | 6 + .../index/mapper/DocumentMapper.java | 57 +- .../index/mapper/DocumentMapperParser.java | 39 +- .../index/mapper/FieldNamesFieldMapper.java | 6 + .../index/mapper/FilterFieldType.java | 23 + .../index/mapper/HllFieldMapper.java | 12 - .../index/mapper/IdFieldMapper.java | 6 + .../index/mapper/IgnoredFieldMapper.java | 6 + .../index/mapper/IndexFieldMapper.java | 6 + .../index/mapper/IpFieldMapper.java | 6 + .../index/mapper/KeywordFieldMapper.java | 85 +- .../index/mapper/MappedFieldType.java | 60 ++ .../org/opensearch/index/mapper/Mapper.java | 52 +- .../index/mapper/MapperService.java | 29 +- .../index/mapper/NestedPathFieldMapper.java | 6 + .../index/mapper/NumberFieldMapper.java | 6 + .../opensearch/index/mapper/ObjectMapper.java | 3 + .../index/mapper/RangeFieldMapper.java | 6 + .../index/mapper/RoutingFieldMapper.java | 15 +- .../mapper/SemanticVersionFieldMapper.java | 9 - .../index/mapper/SeqNoFieldMapper.java | 6 + .../index/mapper/SourceFieldMapper.java | 4 +- .../index/mapper/TextFieldMapper.java | 9 +- ...taFormatPluginAssignCapabilitiesTests.java | 116 +++ .../index/mapper/BinaryFieldMapperTests.java | 8 +- .../mapper/CompletionFieldMapperTests.java | 14 - .../index/mapper/HllFieldMapperTests.java | 61 -- .../index/mapper/KeywordFieldTypeTests.java | 2 +- .../index/mapper/RangeFieldMapperTests.java | 43 - .../SemanticVersionFieldMapperTests.java | 29 - .../mapper/WildcardFieldMapperTests.java | 173 ---- ...AbstractDataFormatAwareEngineTestCase.java | 17 +- .../dataformat/DataFormatTestUtils.java | 34 + .../stub/FileBackedDataFormatPlugin.java | 33 +- .../stub/MockParquetDataFormatPlugin.java | 41 +- 168 files changed, 5568 insertions(+), 2923 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/DeleterImplTests.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LucenePluginBaseTests.java create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractSortedRefreshIT.java create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDataCoherenceIT.java create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldCapabilityIT.java create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldConfigRandomizedIT.java create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedParquetOnlyIT.java delete mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryWithLuceneIT.java delete mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwarePeerRecoveryWithLuceneIT.java delete mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryWithLuceneIT.java delete mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationPromotionWithLuceneIT.java delete mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationWithLuceneIT.java delete mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2WithLuceneIT.java create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/framework/ParquetOnlyDataFormatPlugin.java create mode 100644 sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeAssignCapabilitiesTests.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/NumericParquetField.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IndexParquetField.java create mode 100644 sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetBaseTests.java create mode 100644 sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/parquet/ParquetOnlyDataFormatPlugin.java create mode 100644 server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginAssignCapabilitiesTests.java create mode 100644 test/framework/src/main/java/org/opensearch/index/engine/dataformat/DataFormatTestUtils.java diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index 72d53ee7ed82f..9329d08e03765 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -48,6 +48,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.index.compositeindex.datacube.DimensionType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -252,6 +253,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.POINT_RANGE; + } + @Override public Query termQuery(Object value, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java index ead72b615200a..65181c59c2d31 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -489,11 +489,6 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) { - throw new UnsupportedOperationException(); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java index 9dcbac8edc393..cf4c45dac0c91 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java @@ -178,15 +178,6 @@ protected void parseCreateField(ParseContext context) throws IOException { ); } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - final int tokenCount = parseTokenCount(context); - if (tokenCount == Integer.MIN_VALUE) { - return; - } - context.documentInput().addField(fieldType(), tokenCount); - } - private int parseTokenCount(ParseContext context) throws IOException { final String value; if (context.externalValueSet()) { diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java index 77d1ad63a7964..5bc86477c8e6b 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java @@ -38,8 +38,6 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugins.Plugin; @@ -166,16 +164,4 @@ public void testParseCreateFieldForPluggableFormat() throws Exception { assertNotNull(fieldMapper); assertEquals("rank_feature", fieldMapper.typeName()); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatRankFeatureThrows() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse(source(b -> b.field("field", 10)), docInput) - ); - assertThat(e.getCause(), instanceOf(UnsupportedOperationException.class)); - } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java index 54189c0fa696a..b95572835e612 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java @@ -155,18 +155,4 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage() ); } - - public void testParseCreateFieldForPluggableFormat() throws Exception { - DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - RankFeaturesFieldMapper fieldMapper = (RankFeaturesFieldMapper) mapper.mappers().getMapper("field"); - assertNotNull(fieldMapper); - assertEquals("rank_features", fieldMapper.typeName()); - } - - public void testPluggableDataFormatRankFeaturesThrows() throws IOException { - DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - RankFeaturesFieldMapper rfMapper = (RankFeaturesFieldMapper) mapper.mappers().getMapper("field"); - expectThrows(AssertionError.class, () -> rfMapper.parseCreateFieldForPluggableFormat(null)); - } - } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index ae7dbb12c8cd8..d218f89ff154d 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -52,8 +52,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -771,116 +769,4 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp assertThat(mapper, instanceOf(PrefixFieldMapper.class)); return (PrefixFieldMapper) mapper; } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatSearchAsYouTypeValue() throws Exception { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper( - pluggableSettings, - mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "hello world")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("hello world")); - assertTrue("Expected search_as_you_type value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatSearchAsYouTypeNullSkipped() throws Exception { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper( - pluggableSettings, - mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no field entry for null value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatSearchAsYouTypeExternalValue() throws Exception { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, mapping(b -> { - b.startObject("text_field"); - b.field("type", "text"); - b.startObject("fields"); - b.startObject("sayt").field("type", "search_as_you_type").endObject(); - b.endObject(); - b.endObject(); - })); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("text_field", "external_sayt")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("text_field.sayt") && e.getValue().equals("external_sayt")); - assertTrue("Expected search_as_you_type sub-field captured with external value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatPrefixFieldMapperThrows() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject())); - PrefixFieldMapper prefixMapper = getPrefixFieldMapper(mapper, "field._index_prefix"); - expectThrows(UnsupportedOperationException.class, () -> prefixMapper.parseCreateFieldForPluggableFormat(null)); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatShingleFieldMapperThrows() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject())); - ShingleFieldMapper shingleMapper = getShingleFieldMapper(mapper, "field._2gram"); - expectThrows(UnsupportedOperationException.class, () -> shingleMapper.parseCreateFieldForPluggableFormat(null)); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggablePathEquivalenceWithLucenePath() throws Exception { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - - // Scenario 1: search_as_you_type value - { - DocumentMapper luceneMapper = createDocumentMapper( - mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) - ); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "hello world"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper( - pluggableSettings, - mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.field("field", "hello world")), docInput); - - assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); - assertEquals("hello world", luceneFields[0].stringValue()); - boolean pluggableFound = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("hello world")); - assertTrue("Pluggable path should capture field 'field' with value 'hello world'", pluggableFound); - } - - // Scenario 2: null value — no field produced - { - DocumentMapper luceneMapper = createDocumentMapper( - mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) - ); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper( - pluggableSettings, - mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.nullField("field")), docInput); - - assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); - boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); - } - } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java index a070df84ade9a..a4a612855462f 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java @@ -42,15 +42,11 @@ import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.tests.analysis.Token; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; -import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; import org.opensearch.plugins.Plugin; import java.io.IOException; @@ -70,7 +66,7 @@ public class TokenCountFieldMapperTests extends MapperTestCase { @Override protected Collection getPlugins() { - return List.of(new MapperExtrasModulePlugin(), new MockDataFormatPlugin(), new MockCommitterEnginePlugin()); + return List.of(new MapperExtrasModulePlugin()); } @Override @@ -224,81 +220,4 @@ private SourceToParse createDocument(String fieldValue) throws Exception { private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse request) { return mapper.parse(request).docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed")); } - - private DocumentMapper createIndexWithTokenCountFieldPluggableDataFormat() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - return createDocumentMapper(pluggableSettings, mapping(b -> { - b.startObject("test"); - { - b.field("type", "text"); - b.startObject("fields"); - { - b.startObject("tc"); - { - b.field("type", "token_count"); - b.field("analyzer", "standard"); - } - b.endObject(); - } - b.endObject(); - } - b.endObject(); - })); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatTokenCountValue() throws Exception { - DocumentMapper mapper = createIndexWithTokenCountFieldPluggableDataFormat(); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(createDocument("three tokens string"), docInput); - - boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc") && e.getValue().equals(3)); - assertTrue("Expected token count of 3 for field test.tc", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatTokenCountNullSkipped() throws Exception { - DocumentMapper mapper = createIndexWithTokenCountFieldPluggableDataFormat(); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(createDocument(null), docInput); - - boolean hasTokenCountField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc")); - assertFalse("Expected no token count field for null value", hasTokenCountField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggablePathEquivalenceWithLucenePath() throws Exception { - // Scenario 1: token count value - { - DocumentMapper luceneMapper = createIndexWithTokenCountField(false); - ParseContext.Document luceneDoc = parseDocument(luceneMapper, createDocument("three tokens string")); - IndexableField luceneField = luceneDoc.getField("test.tc"); - - DocumentMapper pluggableMapper = createIndexWithTokenCountFieldPluggableDataFormat(); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(createDocument("three tokens string"), docInput); - - assertNotNull("Lucene path should produce field 'test.tc'", luceneField); - assertEquals(3, luceneField.numericValue()); - boolean pluggableFound = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("test.tc") && e.getValue().equals(3)); - assertTrue("Pluggable path should capture field 'test.tc' with value 3", pluggableFound); - } - - // Scenario 2: null value — no field produced - { - DocumentMapper luceneMapper = createIndexWithTokenCountField(false); - ParseContext.Document luceneDoc = parseDocument(luceneMapper, createDocument(null)); - IndexableField luceneField = luceneDoc.getField("test.tc"); - - DocumentMapper pluggableMapper = createIndexWithTokenCountFieldPluggableDataFormat(); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(createDocument(null), docInput); - - assertNull("Lucene path should produce no field 'test.tc'", luceneField); - boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc")); - assertFalse("Pluggable path should produce no field 'test.tc'", pluggableHasField); - } - } } diff --git a/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java index 72fe63ac83249..e03e1927f3a7d 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java @@ -33,19 +33,14 @@ package org.opensearch.join.mapper; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexService; -import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperException; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.MapperServiceTestCase.CapturingDocumentInput; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.mapper.SourceToParse; import org.opensearch.join.ParentJoinModulePlugin; @@ -58,9 +53,10 @@ import static org.hamcrest.Matchers.containsString; public class ParentJoinFieldMapperTests extends OpenSearchSingleNodeTestCase { + @Override protected Collection> getPlugins() { - return List.of(ParentJoinModulePlugin.class, MockDataFormatPlugin.class, MockCommitterEnginePlugin.class); + return List.of(ParentJoinModulePlugin.class); } public void testSingleLevel() throws Exception { @@ -659,126 +655,4 @@ public void testEagerGlobalOrdinals() throws Exception { assertNotNull(service.mapperService().fieldType("join_field#child")); assertFalse(service.mapperService().fieldType("join_field#child").eagerGlobalOrdinals()); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableFormatParentDoc() throws Exception { - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("join_field") - .field("type", "join") - .startObject("relations") - .field("parent", "child") - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - Settings settings = Settings.builder().put("index.pluggable.dataformat.enabled", true).build(); - IndexService service = createIndex("test", settings); - DocumentMapper docMapper = service.mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - - CapturingDocumentInput docInput = new CapturingDocumentInput(); - Throwable t = expectThrows( - MapperParsingException.class, - () -> docMapper.parse( - new SourceToParse( - "test", - "1", - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("join_field", "parent").endObject()), - MediaTypeRegistry.JSON - ), - docInput - ) - ); - assertEquals(UnsupportedOperationException.class, t.getCause().getClass()); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableFormatChildDoc() throws Exception { - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("join_field") - .field("type", "join") - .startObject("relations") - .field("parent", "child") - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - Settings settings = Settings.builder().put("index.pluggable.dataformat.enabled", true).build(); - IndexService service = createIndex("test", settings); - DocumentMapper docMapper = service.mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - - CapturingDocumentInput docInput = new CapturingDocumentInput(); - Throwable t = expectThrows( - MapperParsingException.class, - () -> docMapper.parse( - new SourceToParse( - "test", - "2", - BytesReference.bytes( - XContentFactory.jsonBuilder() - .startObject() - .startObject("join_field") - .field("name", "child") - .field("parent", "1") - .endObject() - .endObject() - ), - MediaTypeRegistry.JSON, - "1" - ), - docInput - ) - ); - assertEquals(UnsupportedOperationException.class, t.getCause().getClass()); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableFormatParentJoinFieldMapperDirectThrows() throws Exception { - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("join_field") - .field("type", "join") - .startObject("relations") - .field("parent", "child") - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - IndexService service = createIndex("test_direct"); - DocumentMapper docMapper = service.mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - ParentJoinFieldMapper joinMapper = (ParentJoinFieldMapper) docMapper.mappers().getMapper("join_field"); - expectThrows(UnsupportedOperationException.class, () -> joinMapper.parseCreateFieldForPluggableFormat(null)); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableFormatMetaJoinFieldMapperThrows() throws Exception { - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("join_field") - .field("type", "join") - .startObject("relations") - .field("parent", "child") - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - IndexService service = createIndex("test_meta"); - DocumentMapper docMapper = service.mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - MetaJoinFieldMapper metaMapper = (MetaJoinFieldMapper) docMapper.mappers().getMapper("_parent_join"); - assertNotNull(metaMapper); - expectThrows(IllegalStateException.class, () -> metaMapper.parseCreateFieldForPluggableFormat(null)); - } } diff --git a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java index 82a97099a8034..78d85013cf64f 100644 --- a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -810,15 +810,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - final BytesRef binaryValue = parseCollationKey(context); - if (binaryValue == null) { - return; - } - context.documentInput().addField(fieldType(), binaryValue); - } - private BytesRef parseCollationKey(ParseContext context) throws IOException { final String value; if (context.externalValueSet()) { diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index b57352d9e8fb1..c13a8789cd009 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -39,8 +39,6 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugin.analysis.icu.AnalysisICUPlugin; import org.opensearch.plugins.Plugin; @@ -309,61 +307,4 @@ public void testUpdateIgnoreAbove() throws IOException { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatCollationKeyword() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "test_value")), docInput); - - boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Expected ICU collation keyword field captured", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatCollationKeywordNullSkipped() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggablePathEquivalenceWithLucenePath() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - - // Scenario 1: collation keyword value - { - DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "1234"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.field("field", "1234")), docInput); - - assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); - boolean pluggableFound = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Pluggable path should capture field 'field'", pluggableFound); - } - - // Scenario 2: null value — no field produced - { - DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.nullField("field")), docInput); - - assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); - boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); - } - } } diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index e5b97b6f1c045..2655408adc0d3 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -47,8 +47,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -607,85 +605,4 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() { assertThat(e.getMessage(), containsString("Cannot set position_increment_gap on field [field] without positions enabled")); } } - - private Settings pluggableSettings() { - return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatAnnotatedText() throws IOException { - DocumentMapper mapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "some annotated text")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("some annotated text")); - assertTrue("Expected annotated_text field captured with value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatNullValueSkipped() throws IOException { - DocumentMapper mapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatWithExternalValue() throws IOException { - DocumentMapper mapper = createDocumentMapper(pluggableSettings(), mapping(b -> { - b.startObject("text_field"); - b.field("type", "text"); - b.startObject("fields"); - b.startObject("annotated").field("type", "annotated_text").endObject(); - b.endObject(); - b.endObject(); - })); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("text_field", "external_value")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("text_field.annotated") && e.getValue().equals("external_value")); - assertTrue("Expected annotated_text sub-field captured with external value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggablePathEquivalenceWithLucenePath() throws IOException { - // Scenario 1: annotated text value - { - DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "some annotated text"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.field("field", "some annotated text")), docInput); - - assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); - assertEquals("some annotated text", luceneFields[0].stringValue()); - boolean pluggableFound = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("some annotated text")); - assertTrue("Pluggable path should capture field 'field' with value 'some annotated text'", pluggableFound); - } - - // Scenario 2: null value — no field produced - { - DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.nullField("field")), docInput); - - assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); - boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); - } - } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 1e696b77a1a32..2a8b278db859a 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -589,15 +589,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - final String value = getTextValue(context); - if (value == null) { - return; - } - context.documentInput().addField(fieldType(), value); - } - private String getTextValue(ParseContext context) throws IOException { if (context.externalValueSet()) { return context.externalValue().toString(); diff --git a/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 4d88fe8b76087..8c02280b5631b 100644 --- a/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -35,10 +35,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.BytesRef; -import org.opensearch.common.hash.MurmurHash3; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperTestCase; @@ -83,75 +79,4 @@ public void testDefaults() throws Exception { assertEquals(IndexOptions.NONE, field.fieldType().indexOptions()); assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType()); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatMurmur3() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "test_value")), docInput); - - boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Expected murmur3 field captured with hash value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatMurmur3NullSkipped() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggablePathEquivalenceWithLucenePath() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - - // Scenario 1: murmur3 value - { - DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "test_value"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.field("field", "test_value")), docInput); - - assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); - boolean pluggableFound = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Pluggable path should capture field 'field'", pluggableFound); - } - - // Scenario 2: null value — no field produced - { - DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); - - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(b -> b.nullField("field")), docInput); - - assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); - boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); - } - } - - public void testHashCalculation() throws Exception { - String testValue = "test_value"; - BytesRef bytes = new BytesRef(testValue); - long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1; - - // Verify hash is calculated (non-zero for non-empty input) - assertNotEquals("Hash should not be zero for non-empty input", 0L, hash); - - // Verify consistent hashing - BytesRef bytes2 = new BytesRef(testValue); - long hash2 = MurmurHash3.hash128(bytes2.bytes, bytes2.offset, bytes2.length, 0, new MurmurHash3.Hash128()).h1; - assertEquals("Hash should be consistent for same input", hash, hash2); - } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java index 276747da7381a..d2b9b2ee75d4c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java @@ -143,7 +143,7 @@ private void createBankIndex() throws Exception { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); CreateIndexResponse response = client().admin() diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinIT.java index 1476f6e3990a2..ec288e2457a96 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinIT.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinIT.java @@ -154,7 +154,7 @@ private void createParquetBackedIndex(String indexName, String payloadField) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); CreateIndexResponse response = client().admin() diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinMultiNodeIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinMultiNodeIT.java index d24c70d402fba..09ba047523ca1 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinMultiNodeIT.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorJoinMultiNodeIT.java @@ -325,7 +325,7 @@ private void createParquetIndex(String indexName, int numShards, String payloadF .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); CreateIndexResponse response = client().admin() .indices() diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneDataFormat.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneDataFormat.java index 8c9ac5924040c..d1c66093096d7 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneDataFormat.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneDataFormat.java @@ -11,9 +11,25 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.mapper.FieldNamesFieldMapper; +import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.IgnoredFieldMapper; +import org.opensearch.index.mapper.IndexFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MatchOnlyTextFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; +import org.opensearch.index.mapper.RoutingFieldMapper; +import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; +import org.opensearch.index.mapper.TextFieldMapper; import java.util.Set; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.POINT_RANGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.STORED_FIELDS; + /** * {@link DataFormat} descriptor for Lucene inverted indices. *

@@ -33,15 +49,22 @@ public class LuceneDataFormat extends DataFormat { public static final String LUCENE_FORMAT_NAME = "lucene"; private static final Set SUPPORTED_FIELDS = Set.of( - new FieldTypeCapabilities( - "text", - Set.of(FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH, FieldTypeCapabilities.Capability.STORED_FIELDS) - ), - new FieldTypeCapabilities( - "keyword", - Set.of(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, FieldTypeCapabilities.Capability.STORED_FIELDS) - ), - new FieldTypeCapabilities("match_only_text", Set.of(FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH)) + + // Text types — full-text search + stored + new FieldTypeCapabilities(TextFieldMapper.CONTENT_TYPE, Set.of(FULL_TEXT_SEARCH, STORED_FIELDS)), + new FieldTypeCapabilities(KeywordFieldMapper.CONTENT_TYPE, Set.of(FULL_TEXT_SEARCH, STORED_FIELDS, COLUMNAR_STORAGE)), + new FieldTypeCapabilities(MatchOnlyTextFieldMapper.CONTENT_TYPE, Set.of(FULL_TEXT_SEARCH, STORED_FIELDS)), + + // Metadata fields + new FieldTypeCapabilities(SourceFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS)), + new FieldTypeCapabilities(NestedPathFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(FieldNamesFieldMapper.CONTENT_TYPE, Set.of(FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IndexFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IdFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SeqNoFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, POINT_RANGE)), + new FieldTypeCapabilities(SeqNoFieldMapper.PRIMARY_TERM_NAME, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(RoutingFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IgnoredFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)) ); /** {@inheritDoc} Returns {@code "lucene"}. */ diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFieldFactoryRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFieldFactoryRegistry.java index 52e2676c0976c..2c1c47efaefdf 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFieldFactoryRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFieldFactoryRegistry.java @@ -10,13 +10,16 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.util.BytesRef; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MatchOnlyTextFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; import org.opensearch.index.mapper.TextFieldMapper; import java.util.Map; @@ -35,6 +38,17 @@ @ExperimentalApi public final class LuceneFieldFactoryRegistry { + private static final FieldType ID_FIELD_TYPE = new FieldType(); + + static { + ID_FIELD_TYPE.setTokenized(false); + ID_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + ID_FIELD_TYPE.setOmitNorms(true); + ID_FIELD_TYPE.setStored(false); + ID_FIELD_TYPE.setDocValuesType(DocValuesType.NONE); + ID_FIELD_TYPE.freeze(); + } + // ── Default factories ── private static final LuceneFieldFactory TEXT_FACTORY = (doc, ft, value, lft) -> { doc.add(new Field(ft.name(), value.toString(), lft)); @@ -48,14 +62,12 @@ public final class LuceneFieldFactoryRegistry { doc.add(new Field(ft.name(), value.toString(), lft)); }; - private static final FieldType ID_FIELD_TYPE = buildIdFieldType(); - private static final LuceneFieldFactory ID_FIELD_FACTORY = (doc, ft, value, lft) -> { doc.add(new Field(ft.name(), new BytesRef((byte[]) value), ID_FIELD_TYPE)); }; private static final LuceneFieldFactory SEQ_NO_FIELD_FACTORY = (doc, ft, value, lft) -> { - doc.add(new LongPoint(ft.name(), (long) value)); + // do nothing for now since we don't want to index seq no indexing without soft deletes enabled. }; // ── Registry ── @@ -69,8 +81,15 @@ public LuceneFieldFactoryRegistry() { register(TextFieldMapper.CONTENT_TYPE, TEXT_FACTORY); register(KeywordFieldMapper.CONTENT_TYPE, KEYWORD_FACTORY); register(MatchOnlyTextFieldMapper.CONTENT_TYPE, MATCH_ONLY_TEXT_FACTORY); + registerMetaFields(); + } + + private void registerMetaFields() { register(IdFieldMapper.CONTENT_TYPE, ID_FIELD_FACTORY); register(SeqNoFieldMapper.CONTENT_TYPE, SEQ_NO_FIELD_FACTORY); + register(SeqNoFieldMapper.PRIMARY_TERM_NAME, (d, ft, v, lft) -> d.add(new SortedNumericDocValuesField(ft.name(), (long) v))); + register(SourceFieldMapper.CONTENT_TYPE, (d, ft, v, lft) -> d.add(new Field(ft.name(), (BytesRef) v, lft))); + // pending routing and ignored field handling } /** @@ -101,11 +120,4 @@ public LuceneFieldFactory get(String typeName) { public Set supportedTypes() { return Set.copyOf(factories.keySet()); } - - private static FieldType buildIdFieldType() { - FieldType ft = new FieldType(IdFieldMapper.Defaults.FIELD_TYPE); - ft.setStored(false); - ft.freeze(); - return ft; - } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java index da66ac5d0b349..26d62203c8ec7 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java @@ -53,7 +53,7 @@ @ExperimentalApi public class LucenePlugin extends Plugin implements DataFormatPlugin, SearchBackEndPlugin, EnginePlugin { - private static final LuceneDataFormat DATA_FORMAT = new LuceneDataFormat(); + public static final LuceneDataFormat DATA_FORMAT = new LuceneDataFormat(); /** Creates a new LucenePlugin. */ public LucenePlugin() {} diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java index 9ddc233167248..07448774f32f0 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java @@ -14,10 +14,14 @@ import org.apache.lucene.index.DocValuesType; import org.opensearch.be.lucene.LuceneFieldFactory; import org.opensearch.be.lucene.LuceneFieldFactoryRegistry; +import org.opensearch.be.lucene.LucenePlugin; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.dataformat.DocumentInput; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; +import java.util.Set; + /** * Lucene-specific {@link DocumentInput} that builds a Lucene {@link Document}. * @@ -71,28 +75,52 @@ public Document getFinalInput() { /** * Adds a field to the underlying Lucene document by looking up the appropriate * {@link LuceneFieldFactory} from the registry based on the field's type name. - * Silently skips null values, null field types, and unregistered type names. + *

+ * The field is accepted only if OWNING_FORMAT owns at least one capability + * for this field according to {@link MappedFieldType#getCapabilityMap()}. Fields with + * an empty capability map (no format declared support) and fields owned by other + * formats are silently skipped, mirroring the per-format self-filtering used by + * {@code ParquetDocumentInput}. * * @param fieldType the OpenSearch mapped field type * @param value the field value */ @Override public void addField(MappedFieldType fieldType, Object value) { - assert value != null : "Field value must not be null"; + Set capabilities = fieldType.getCapabilityMap().getOrDefault(LucenePlugin.DATA_FORMAT, Set.of()); + if (capabilities.isEmpty()) { + // nothing to support on this format for this field. + return; + } + if (value == null) { + throw new IllegalArgumentException( + "Field value must not be null for: " + fieldType.name() + " of type: " + fieldType.typeName() + ); + } LuceneFieldFactory factory = fieldFactory(fieldType); if (factory == null) { - return; + // capabilities need to be supported but actual implementation to support lucene field type does not exist. + throw new IllegalArgumentException( + "Field: " + fieldType.name() + " requests capability: " + capabilities + " but does not have any factory to support" + ); } - FieldType luceneFieldType; + FieldType luceneFieldType = getFieldType(fieldType, capabilities); + factory.addField(document, fieldType, value, luceneFieldType); + } + + private static FieldType getFieldType(MappedFieldType fieldType, Set capabilities) { + FieldType luceneFieldType = null; if (fieldType.getTextSearchInfo() != null && fieldType.getTextSearchInfo().getLuceneFieldType() != null) { luceneFieldType = new FieldType(fieldType.getTextSearchInfo().getLuceneFieldType()); - luceneFieldType.setDocValuesType(DocValuesType.NONE); + if (!capabilities.contains(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE)) { + // Disable doc values even if core mappers have set it on lucene fields + // once we introduce more frontend params, we can remove this check. + luceneFieldType.setDocValuesType(DocValuesType.NONE); + } luceneFieldType.setStored(false); luceneFieldType.setOmitNorms(true); - } else { - luceneFieldType = null; } - factory.addField(document, fieldType, value, luceneFieldType); + return luceneFieldType; } private LuceneFieldFactory fieldFactory(MappedFieldType fieldType) { diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/DeleterImplTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/DeleterImplTests.java new file mode 100644 index 0000000000000..2b5b6b538c7fb --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/DeleterImplTests.java @@ -0,0 +1,820 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DeleterImplTests extends OpenSearchTestCase { + + private Writer mockWriter; + private DeleterImpl> deleter; + + @Override + public void setUp() throws Exception { + super.setUp(); + mockWriter = mock(Writer.class); + when(mockWriter.generation()).thenReturn(1L); + deleter = new DeleterImpl<>(mockWriter); + } + + // ===== Basic functionality tests ===== + + public void testBasicFunctionality() { + assertEquals(1L, deleter.generation()); + assertTrue(deleter.isActive()); + } + + public void testGenerationMatchesWriter() { + Writer writer = mock(Writer.class); + when(writer.generation()).thenReturn(42L); + DeleterImpl> d = new DeleterImpl<>(writer); + assertEquals(42L, d.generation()); + } + + public void testDeleteDoc() throws IOException { + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + DeleteResult expectedResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(deleteInput)).thenReturn(expectedResult); + + DeleteResult result = deleter.deleteDoc(deleteInput); + + assertEquals(expectedResult, result); + verify(mockWriter).deleteDocument(deleteInput); + } + + public void testDeleteDocReturnsFailureResult() throws IOException { + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + DeleteResult failureResult = new DeleteResult.Failure(new RuntimeException("oops")); + when(mockWriter.deleteDocument(deleteInput)).thenReturn(failureResult); + + DeleteResult result = deleter.deleteDoc(deleteInput); + + assertEquals(failureResult, result); + } + + public void testDeleteDocWhenInactive() throws IOException { + deleter.deactivate(); + + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + + DeleteResult result = deleter.deleteDoc(deleteInput); + + assertNull(result); + } + + public void testDeleteDocMultipleCalls() throws IOException { + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenReturn(mockResult); + + for (int i = 0; i < 10; i++) { + DeleteInput deleteInput = new DeleteInput("_id", "doc" + i, 1L); + DeleteResult result = deleter.deleteDoc(deleteInput); + assertNotNull(result); + assertTrue(result instanceof DeleteResult.Success); + } + + verify(mockWriter, times(10)).deleteDocument(any(DeleteInput.class)); + } + + public void testRecordBufferedDeletes() { + assertTrue(deleter.recordBufferedDeletes("doc1")); + assertTrue(deleter.recordBufferedDeletes("doc2")); + assertTrue(deleter.isActive()); + } + + public void testRecordBufferedDeletesDuplicateIds() { + assertTrue(deleter.recordBufferedDeletes("doc1")); + assertTrue(deleter.recordBufferedDeletes("doc1")); + + Queue bufferedDeletes = deleter.deactivate(); + assertEquals(2, bufferedDeletes.size()); + } + + public void testRecordBufferedDeletesWhenInactive() { + deleter.deactivate(); + + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> deleter.recordBufferedDeletes("doc1")); + + assertTrue(exception.getMessage().contains("Cannot record a delete on a closed deleter")); + } + + public void testDeactivate() { + deleter.recordBufferedDeletes("doc1"); + deleter.recordBufferedDeletes("doc2"); + + Queue bufferedDeletes = deleter.deactivate(); + + assertFalse(deleter.isActive()); + assertEquals(2, bufferedDeletes.size()); + assertTrue(bufferedDeletes.contains("doc1")); + assertTrue(bufferedDeletes.contains("doc2")); + } + + public void testDeactivatePreservesOrder() { + deleter.recordBufferedDeletes("first"); + deleter.recordBufferedDeletes("second"); + deleter.recordBufferedDeletes("third"); + + Queue bufferedDeletes = deleter.deactivate(); + + assertEquals("first", bufferedDeletes.poll()); + assertEquals("second", bufferedDeletes.poll()); + assertEquals("third", bufferedDeletes.poll()); + } + + public void testDeactivateWhenAlreadyInactive() { + deleter.deactivate(); + + Queue bufferedDeletes = deleter.deactivate(); + + assertFalse(deleter.isActive()); + assertTrue(bufferedDeletes.isEmpty()); + } + + public void testDeactivateWithNoBufferedDeletes() { + Queue bufferedDeletes = deleter.deactivate(); + + assertFalse(deleter.isActive()); + assertTrue(bufferedDeletes.isEmpty()); + } + + public void testClose() throws IOException { + deleter.recordBufferedDeletes("doc1"); + assertTrue(deleter.isActive()); + + deleter.close(); + + assertFalse(deleter.isActive()); + } + + public void testCloseIsIdempotent() throws IOException { + deleter.close(); + deleter.close(); + assertFalse(deleter.isActive()); + } + + public void testDeleteDocAfterClose() throws IOException { + deleter.close(); + + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + DeleteResult result = deleter.deleteDoc(deleteInput); + + assertNull(result); + } + + // ===== Exception handling tests ===== + + public void testExceptionHandlingInDeleteDoc() throws IOException { + IOException expectedException = new IOException("Test exception"); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenThrow(expectedException); + + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + + IOException thrownException = expectThrows(IOException.class, () -> deleter.deleteDoc(deleteInput)); + + assertEquals(expectedException, thrownException); + assertTrue(deleter.isActive()); + } + + public void testIOExceptionDoesNotAffectBufferedDeletes() throws IOException { + IOException expectedException = new IOException("disk error"); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenThrow(expectedException); + + deleter.recordBufferedDeletes("doc1"); + + try { + deleter.deleteDoc(new DeleteInput("_id", "doc2", 1L)); + fail("Expected IOException"); + } catch (IOException e) { + // expected + } + + deleter.recordBufferedDeletes("doc3"); + assertTrue(deleter.isActive()); + + Queue buffered = deleter.deactivate(); + assertEquals(2, buffered.size()); + assertTrue(buffered.contains("doc1")); + assertTrue(buffered.contains("doc3")); + } + + public void testRuntimeExceptionFromWriterDoesNotDeactivate() throws IOException { + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenThrow(new RuntimeException("unexpected")); + + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + + expectThrows(RuntimeException.class, () -> deleter.deleteDoc(deleteInput)); + + assertTrue(deleter.isActive()); + assertTrue(deleter.recordBufferedDeletes("doc2")); + } + + // ===== Multi-threading tests ===== + + public void testConcurrentDeleteOperations() throws Exception { + int numThreads = 10; + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch completeLatch = new CountDownLatch(numThreads); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger failureCount = new AtomicInteger(0); + + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenReturn(mockResult); + + for (int i = 0; i < numThreads; i++) { + final int threadId = i; + executor.submit(() -> { + try { + startLatch.await(); + DeleteInput deleteInput = new DeleteInput("_id", "doc" + threadId, 1L); + DeleteResult result = deleter.deleteDoc(deleteInput); + if (result != null) { + successCount.incrementAndGet(); + } else { + failureCount.incrementAndGet(); + } + } catch (Exception e) { + failureCount.incrementAndGet(); + } finally { + completeLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(completeLatch.await(10, TimeUnit.SECONDS)); + + assertEquals(numThreads, successCount.get()); + assertEquals(0, failureCount.get()); + + executor.shutdown(); + } + + public void testConcurrentBufferedDeleteRecording() throws Exception { + int numThreads = 20; + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch completeLatch = new CountDownLatch(numThreads); + AtomicInteger successCount = new AtomicInteger(0); + + for (int i = 0; i < numThreads; i++) { + final int threadId = i; + executor.submit(() -> { + try { + startLatch.await(); + boolean success = deleter.recordBufferedDeletes("doc" + threadId); + if (success) { + successCount.incrementAndGet(); + } + } catch (Exception e) { + // Expected for some threads if deleter becomes inactive + } finally { + completeLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(completeLatch.await(10, TimeUnit.SECONDS)); + + assertEquals(numThreads, successCount.get()); + + Queue bufferedDeletes = deleter.deactivate(); + assertEquals(numThreads, bufferedDeletes.size()); + + executor.shutdown(); + } + + public void testConcurrentDeleteAndDeactivate() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(3); + CyclicBarrier barrier = new CyclicBarrier(3); + AtomicReference exception = new AtomicReference<>(); + AtomicBoolean deactivateCompleted = new AtomicBoolean(false); + + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenReturn(mockResult); + + // Delete thread + Future deleteFuture = executor.submit(() -> { + try { + barrier.await(); + for (int i = 0; i < 100; i++) { + DeleteInput deleteInput = new DeleteInput("_id", "doc" + i, 1L); + deleter.deleteDoc(deleteInput); + Thread.yield(); + } + } catch (Exception e) { + exception.set(e); + } + }); + + // Buffered delete recording thread + Future bufferFuture = executor.submit(() -> { + try { + barrier.await(); + for (int i = 0; i < 50; i++) { + try { + deleter.recordBufferedDeletes("buffered" + i); + } catch (IllegalStateException e) { + // Expected when deleter becomes inactive + break; + } + Thread.yield(); + } + } catch (Exception e) { + exception.set(e); + } + }); + + // Deactivate thread + Future deactivateFuture = executor.submit(() -> { + try { + barrier.await(); + Thread.sleep(50); // Let other threads run first + deleter.deactivate(); + deactivateCompleted.set(true); + } catch (Exception e) { + exception.set(e); + } + }); + + deleteFuture.get(10, TimeUnit.SECONDS); + bufferFuture.get(10, TimeUnit.SECONDS); + deactivateFuture.get(10, TimeUnit.SECONDS); + + assertTrue(deactivateCompleted.get()); + assertFalse(deleter.isActive()); + assertNull(exception.get()); + + executor.shutdown(); + } + + public void testConcurrentMultipleDeactivations() throws Exception { + // Add some buffered deletes first + deleter.recordBufferedDeletes("doc1"); + deleter.recordBufferedDeletes("doc2"); + deleter.recordBufferedDeletes("doc3"); + + int numThreads = 5; + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch completeLatch = new CountDownLatch(numThreads); + List> results = Collections.synchronizedList(new ArrayList<>()); + + for (int i = 0; i < numThreads; i++) { + executor.submit(() -> { + try { + startLatch.await(); + Queue result = deleter.deactivate(); + results.add(result); + } catch (Exception e) { + // Unexpected + } finally { + completeLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(completeLatch.await(10, TimeUnit.SECONDS)); + + // Only one thread should get the buffered deletes, others should get empty queues + int nonEmptyResults = 0; + int totalDeletes = 0; + for (Queue result : results) { + if (!result.isEmpty()) { + nonEmptyResults++; + totalDeletes += result.size(); + } + } + + assertEquals(1, nonEmptyResults); + assertEquals(3, totalDeletes); + assertFalse(deleter.isActive()); + + executor.shutdown(); + } + + public void testConcurrentReadWriteLockContention() throws Exception { + int numReaders = 10; + int numWriters = 2; + ExecutorService executor = Executors.newFixedThreadPool(numReaders + numWriters); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch completeLatch = new CountDownLatch(numReaders + numWriters); + AtomicInteger readerSuccessCount = new AtomicInteger(0); + AtomicInteger writerSuccessCount = new AtomicInteger(0); + + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenReturn(mockResult); + + // Reader threads (deleteDoc and recordBufferedDeletes) + for (int i = 0; i < numReaders; i++) { + final int threadId = i; + executor.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < 10; j++) { + if (threadId % 2 == 0) { + // Delete operation + DeleteInput deleteInput = new DeleteInput("_id", "doc" + threadId + "_" + j, 1L); + DeleteResult result = deleter.deleteDoc(deleteInput); + if (result != null) { + readerSuccessCount.incrementAndGet(); + } + } else { + // Buffer delete operation + try { + boolean success = deleter.recordBufferedDeletes("buffered" + threadId + "_" + j); + if (success) { + readerSuccessCount.incrementAndGet(); + } + } catch (IllegalStateException e) { + // Expected when deleter becomes inactive + } + } + Thread.yield(); + } + } catch (Exception e) { + // Some failures expected + } finally { + completeLatch.countDown(); + } + }); + } + + // Writer threads (deactivate) + for (int i = 0; i < numWriters; i++) { + executor.submit(() -> { + try { + startLatch.await(); + Thread.sleep(100); // Let readers run first + deleter.deactivate(); + writerSuccessCount.incrementAndGet(); + } catch (Exception e) { + // Unexpected + } finally { + completeLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(completeLatch.await(15, TimeUnit.SECONDS)); + + // At least one writer should succeed + assertTrue(writerSuccessCount.get() >= 1); + assertFalse(deleter.isActive()); + + executor.shutdown(); + } + + public void testStateConsistencyAfterConcurrentOperations() throws Exception { + int numOperations = 1000; + ExecutorService executor = Executors.newFixedThreadPool(4); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch completeLatch = new CountDownLatch(4); + + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenReturn(mockResult); + + // Mixed operations thread + executor.submit(() -> { + try { + startLatch.await(); + for (int i = 0; i < numOperations / 4; i++) { + if (deleter.isActive()) { + try { + deleter.recordBufferedDeletes("mixed" + i); + } catch (IllegalStateException e) { + break; + } + DeleteInput deleteInput = new DeleteInput("_id", "mixed" + i, 1L); + deleter.deleteDoc(deleteInput); + } + if (i % 100 == 0) Thread.yield(); + } + } catch (Exception e) { + // Expected when deleter becomes inactive + } finally { + completeLatch.countDown(); + } + }); + + // Buffer-only thread + executor.submit(() -> { + try { + startLatch.await(); + for (int i = 0; i < numOperations / 4; i++) { + try { + deleter.recordBufferedDeletes("buffer" + i); + } catch (IllegalStateException e) { + break; // Deleter became inactive + } + if (i % 100 == 0) Thread.yield(); + } + } catch (Exception e) { + // Unexpected + } finally { + completeLatch.countDown(); + } + }); + + // Delete-only thread + executor.submit(() -> { + try { + startLatch.await(); + for (int i = 0; i < numOperations / 4; i++) { + if (deleter.isActive()) { + DeleteInput deleteInput = new DeleteInput("_id", "delete" + i, 1L); + deleter.deleteDoc(deleteInput); + } + if (i % 100 == 0) Thread.yield(); + } + } catch (Exception e) { + // Expected when deleter becomes inactive + } finally { + completeLatch.countDown(); + } + }); + + // Deactivation thread + executor.submit(() -> { + try { + startLatch.await(); + Thread.sleep(200); // Let other threads run + Queue bufferedDeletes = deleter.deactivate(); + assertNotNull(bufferedDeletes); + } catch (Exception e) { + // Unexpected + } finally { + completeLatch.countDown(); + } + }); + + startLatch.countDown(); + assertTrue(completeLatch.await(30, TimeUnit.SECONDS)); + + // Final state should be inactive + assertFalse(deleter.isActive()); + + // Any subsequent operations should fail appropriately + expectThrows(IllegalStateException.class, () -> deleter.recordBufferedDeletes("final")); + assertNull(deleter.deleteDoc(new DeleteInput("_id", "final", 1L))); + + executor.shutdown(); + } + + // ===== Race condition: exactly-once draining guarantee ===== + + public void testBufferedDeletesNotLostDuringConcurrentDeactivation() throws Exception { + int numThreads = 8; + int deletesPerThread = 50; + ExecutorService executor = Executors.newFixedThreadPool(numThreads + 1); + CyclicBarrier barrier = new CyclicBarrier(numThreads + 1); + Set successfullyRecorded = ConcurrentHashMap.newKeySet(); + AtomicReference> drainedRef = new AtomicReference<>(); + + for (int t = 0; t < numThreads; t++) { + final int threadId = t; + executor.submit(() -> { + try { + barrier.await(); + for (int i = 0; i < deletesPerThread; i++) { + String id = "t" + threadId + "_d" + i; + try { + deleter.recordBufferedDeletes(id); + successfullyRecorded.add(id); + } catch (IllegalStateException e) { + break; + } + } + } catch (Exception e) { + // Expected + } + }); + } + + executor.submit(() -> { + try { + barrier.await(); + Thread.sleep(30); + drainedRef.set(deleter.deactivate()); + } catch (Exception e) { + // Expected + } + }); + + executor.shutdown(); + assertTrue(executor.awaitTermination(15, TimeUnit.SECONDS)); + + Queue drained = drainedRef.get(); + assertNotNull(drained); + + Set drainedSet = new HashSet<>(drained); + for (String id : drainedSet) { + assertTrue("Drained id '" + id + "' was not successfully recorded", successfullyRecorded.contains(id)); + } + + for (String id : successfullyRecorded) { + assertTrue("Successfully recorded id '" + id + "' was lost during deactivation", drainedSet.contains(id)); + } + } + + public void testDeactivateUnderHighWriteContention() throws Exception { + int numWriters = 16; + int writesPerWriter = 100; + ExecutorService executor = Executors.newFixedThreadPool(numWriters + 1); + CyclicBarrier barrier = new CyclicBarrier(numWriters + 1); + AtomicInteger totalRecorded = new AtomicInteger(0); + AtomicReference> drainedRef = new AtomicReference<>(); + + for (int t = 0; t < numWriters; t++) { + final int threadId = t; + executor.submit(() -> { + try { + barrier.await(); + for (int i = 0; i < writesPerWriter; i++) { + try { + deleter.recordBufferedDeletes("w" + threadId + "_" + i); + totalRecorded.incrementAndGet(); + } catch (IllegalStateException e) { + break; + } + } + } catch (Exception e) { + // Expected + } + }); + } + + executor.submit(() -> { + try { + barrier.await(); + Thread.sleep(50); + drainedRef.set(deleter.deactivate()); + } catch (Exception e) { + // Expected + } + }); + + executor.shutdown(); + assertTrue(executor.awaitTermination(15, TimeUnit.SECONDS)); + + Queue drained = drainedRef.get(); + assertNotNull(drained); + assertEquals(totalRecorded.get(), drained.size()); + } + + // ===== Race condition: close while deleteDoc is in flight ===== + + public void testCloseWhileDeleteDocInFlight() throws Exception { + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenAnswer(invocation -> { + Thread.sleep(50); + return mockResult; + }); + + ExecutorService executor = Executors.newFixedThreadPool(2); + CyclicBarrier barrier = new CyclicBarrier(2); + AtomicReference deleteResult = new AtomicReference<>(); + + executor.submit(() -> { + try { + barrier.await(); + DeleteInput deleteInput = new DeleteInput("_id", "doc1", 1L); + deleteResult.set(deleter.deleteDoc(deleteInput)); + } catch (Exception e) { + // Expected + } + }); + + executor.submit(() -> { + try { + barrier.await(); + Thread.sleep(10); + deleter.close(); + } catch (Exception e) { + // Expected + } + }); + + executor.shutdown(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + + assertFalse(deleter.isActive()); + } + + // ===== Race condition: deleteDoc returns null after deactivation ===== + + public void testDeleteDocReturnsNullAfterDeactivation() throws Exception { + DeleteResult mockResult = new DeleteResult.Success(1L, 1L, 1L); + when(mockWriter.deleteDocument(any(DeleteInput.class))).thenReturn(mockResult); + + ExecutorService executor = Executors.newFixedThreadPool(2); + CyclicBarrier barrier = new CyclicBarrier(2); + AtomicInteger nullResults = new AtomicInteger(0); + AtomicInteger successResults = new AtomicInteger(0); + + executor.submit(() -> { + try { + barrier.await(); + for (int i = 0; i < 200; i++) { + DeleteInput deleteInput = new DeleteInput("_id", "doc" + i, 1L); + DeleteResult result = deleter.deleteDoc(deleteInput); + if (result == null) { + nullResults.incrementAndGet(); + } else { + successResults.incrementAndGet(); + } + } + } catch (Exception e) { + // Expected + } + }); + + executor.submit(() -> { + try { + barrier.await(); + Thread.sleep(5); + deleter.deactivate(); + } catch (Exception e) { + // Expected + } + }); + + executor.shutdown(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + + // All 200 operations must complete — some succeed, some return null after deactivation + assertEquals(200, successResults.get() + nullResults.get()); + assertTrue(successResults.get() > 0); + // After deactivation, subsequent calls must return null + assertFalse(deleter.isActive()); + assertNull(deleter.deleteDoc(new DeleteInput("_id", "final_check", 1L))); + } + + // ===== High-volume stress test ===== + + public void testHighVolumeBufferedDeletesThenDrain() { + int numDeletes = 10_000; + for (int i = 0; i < numDeletes; i++) { + assertTrue(deleter.recordBufferedDeletes("doc" + i)); + } + + Queue drained = deleter.deactivate(); + assertEquals(numDeletes, drained.size()); + + Set seen = new HashSet<>(); + for (String id : drained) { + assertTrue("Duplicate id found: " + id, seen.add(id)); + } + } + + public void testRepeatedCloseDeactivateCycles() throws IOException { + for (int cycle = 0; cycle < 5; cycle++) { + Writer w = mock(Writer.class); + when(w.generation()).thenReturn((long) cycle); + DeleterImpl> d = new DeleterImpl<>(w); + + d.recordBufferedDeletes("doc" + cycle); + assertTrue(d.isActive()); + + d.close(); + assertFalse(d.isActive()); + + Queue drained = d.deactivate(); + assertTrue(drained.isEmpty()); + } + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneDocumentInputTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneDocumentInputTests.java index e16fe44f02a29..fb79ba2e51f09 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneDocumentInputTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneDocumentInputTests.java @@ -9,23 +9,20 @@ package org.opensearch.be.lucene.index; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.opensearch.be.lucene.LucenePlugin; import org.opensearch.index.mapper.IdFieldMapper; -import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.MatchOnlyTextFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.TextFieldMapper; -import org.opensearch.index.mapper.TextSearchInfo; -import org.opensearch.test.OpenSearchTestCase; import java.nio.charset.StandardCharsets; -import java.util.Collections; +import java.util.Map; +import java.util.Set; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -33,7 +30,7 @@ * Verifies that each field type registered in {@link org.opensearch.be.lucene.LuceneFieldFactoryRegistry} * produces Lucene fields with the expected storage properties. */ -public class LuceneDocumentInputTests extends OpenSearchTestCase { +public class LuceneDocumentInputTests extends LucenePluginBaseTests { public void testIdFieldProperties() { MappedFieldType idField = mockIdField(); @@ -50,7 +47,7 @@ public void testIdFieldProperties() { } public void testTextFieldProperties() { - MappedFieldType textField = new TextFieldMapper.TextFieldType("content"); + MappedFieldType textField = mockTextField("content"); LuceneDocumentInput input = new LuceneDocumentInput(); input.addField(textField, "hello world"); @@ -66,13 +63,7 @@ public void testTextFieldProperties() { } public void testKeywordFieldProperties() { - FieldType kwFieldType = new FieldType(); - kwFieldType.setTokenized(false); - kwFieldType.setStored(false); - kwFieldType.setOmitNorms(true); - kwFieldType.setIndexOptions(IndexOptions.DOCS); - kwFieldType.freeze(); - MappedFieldType keywordField = new KeywordFieldMapper.KeywordFieldType("status", kwFieldType); + MappedFieldType keywordField = mockKeywordField("status"); LuceneDocumentInput input = new LuceneDocumentInput(); input.addField(keywordField, "active"); @@ -89,13 +80,7 @@ public void testKeywordFieldProperties() { } public void testMatchOnlyTextFieldProperties() { - MappedFieldType matchOnlyField = new MatchOnlyTextFieldMapper.MatchOnlyTextFieldType( - "body", - true, - false, - new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, null, null), - Collections.emptyMap() - ); + MappedFieldType matchOnlyField = mockMatchOnlyTextField("body"); LuceneDocumentInput input = new LuceneDocumentInput(); input.addField(matchOnlyField, "some text"); @@ -108,7 +93,7 @@ public void testMatchOnlyTextFieldProperties() { assertFalse("match_only_text: should not be stored", ft.stored()); assertTrue("match_only_text: should omit norms", ft.omitNorms()); assertEquals("match_only_text: should have no doc values", DocValuesType.NONE, ft.docValuesType()); - assertNotEquals("match_only_text: should be indexed", IndexOptions.NONE, ft.indexOptions()); + assertNotEquals("match_only_text: should be indexed", IndexOptions.DOCS, ft.indexOptions()); } public void testSeqNoFieldProperties() { @@ -118,19 +103,14 @@ public void testSeqNoFieldProperties() { Document doc = input.getFinalInput(); IndexableField field = doc.getField(SeqNoFieldMapper.NAME); - assertNotNull("_seq_no field should be present in document", field); - - // LongPoint: dimensional field, not stored, not indexed via inverted index - IndexableFieldType ft = field.fieldType(); - assertFalse("_seq_no: should not be stored", ft.stored()); - assertEquals("_seq_no: LongPoint has no inverted index", IndexOptions.NONE, ft.indexOptions()); - assertTrue("_seq_no: should have point dimensions", ft.pointDimensionCount() > 0); + assertNull("_seq_no field should be present in document", field); } private static MappedFieldType mockIdField() { MappedFieldType idField = mock(MappedFieldType.class); when(idField.typeName()).thenReturn(IdFieldMapper.CONTENT_TYPE); when(idField.name()).thenReturn(IdFieldMapper.NAME); + when(idField.getCapabilityMap()).thenReturn(Map.of(LucenePlugin.DATA_FORMAT, Set.of(FULL_TEXT_SEARCH))); return idField; } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java index c0e1cd0237e79..208e486dc61e0 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java @@ -10,8 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.tests.analysis.MockAnalyzer; @@ -34,10 +32,8 @@ import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.commit.CommitterConfig; -import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.TextFieldMapper.TextFieldType; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.ShardPath; @@ -49,7 +45,6 @@ import org.opensearch.plugins.PluginsService; import org.opensearch.test.DummyShardLock; import org.opensearch.test.IndexSettingsModule; -import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.nio.file.Files; @@ -64,7 +59,7 @@ /** * Tests for {@link LuceneIndexingExecutionEngine}. */ -public class LuceneIndexingExecutionEngineTests extends OpenSearchTestCase { +public class LuceneIndexingExecutionEngineTests extends LucenePluginBaseTests { private LuceneCommitter committer; private Store store; @@ -172,7 +167,7 @@ public void testRefreshIncorporatesLuceneSegments() throws IOException { // Use LuceneWriter to create segments (which sets the writer_generation attribute via LuceneWriterCodec) Path tempBase = createTempDir(); - MappedFieldType textField = new TextFieldType("content"); + MappedFieldType textField = mockTextField("content"); long generation = 1L; try ( @@ -281,14 +276,8 @@ public void testWriterCreationAndFlushThroughEngine() throws IOException { int numDocs = randomIntBetween(3, 15); long generation = 1L; - MappedFieldType textField = new TextFieldType("content"); - final FieldType keywordFieldType = new FieldType(); - keywordFieldType.setTokenized(false); - keywordFieldType.setStored(false); - keywordFieldType.setOmitNorms(true); - keywordFieldType.setIndexOptions(IndexOptions.DOCS); - keywordFieldType.freeze(); - MappedFieldType keywordField = new KeywordFieldType("tag", keywordFieldType); + MappedFieldType textField = mockTextField("content"); + MappedFieldType keywordField = mockKeywordField("tag"); // Create writer through the engine Writer writer = engine.createWriter(new WriterConfig(generation)); @@ -336,7 +325,7 @@ public void testMultipleWriterRefreshAccumulatesInSharedWriter() throws IOExcept LuceneIndexingExecutionEngine engine = new LuceneIndexingExecutionEngine(luceneDataFormat, committer, mapperService, store); IndexWriter sharedWriter = committer.getIndexWriter(); - MappedFieldType textField = new TextFieldType("content"); + MappedFieldType textField = mockTextField("content"); long gen1 = 1L; long gen2 = 2L; @@ -445,7 +434,7 @@ public void testGetHeapBytesUsedSumsActiveWriters() throws IOException { // Create writers via engine.createWriter to use the engine's internal activeWriters set Path tempBase = createTempDir(); - MappedFieldType textField = new TextFieldType("content"); + MappedFieldType textField = mockTextField("content"); WriterConfig config1 = new WriterConfig(1L); WriterConfig config2 = new WriterConfig(2L); diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LucenePluginBaseTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LucenePluginBaseTests.java new file mode 100644 index 0000000000000..c373dab9d8d71 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LucenePluginBaseTests.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene.index; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MatchOnlyTextFieldMapper; +import org.opensearch.index.mapper.TextFieldMapper; +import org.opensearch.index.mapper.TextSearchInfo; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.Set; + +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + +public abstract class LucenePluginBaseTests extends OpenSearchTestCase { + + protected MappedFieldType mockTextField(String name) { + TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType(name); + textFieldType.setCapabilityMap(Map.of(LucenePlugin.DATA_FORMAT, Set.of(FULL_TEXT_SEARCH))); + return textFieldType; + } + + protected MappedFieldType mockMatchOnlyTextField(String name) { + TextFieldMapper.TextFieldType textFieldType = new MatchOnlyTextFieldMapper.MatchOnlyTextFieldType( + name, + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + Map.of() + ); + textFieldType.setCapabilityMap(Map.of(LucenePlugin.DATA_FORMAT, Set.of(FULL_TEXT_SEARCH))); + return textFieldType; + } + + protected MappedFieldType mockKeywordField(String name) { + final FieldType keywordFieldType = new FieldType(); + keywordFieldType.setTokenized(false); + keywordFieldType.setStored(false); + keywordFieldType.setOmitNorms(true); + keywordFieldType.setIndexOptions(IndexOptions.DOCS); + keywordFieldType.freeze(); + KeywordFieldMapper.KeywordFieldType kft = new KeywordFieldMapper.KeywordFieldType(name, keywordFieldType); + kft.setCapabilityMap(Map.of(LucenePlugin.DATA_FORMAT, Set.of(FULL_TEXT_SEARCH))); + return kft; + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterSortedFlushTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterSortedFlushTests.java index c95914fef3b3c..cc196068f0727 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterSortedFlushTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterSortedFlushTests.java @@ -24,8 +24,6 @@ import org.opensearch.index.engine.dataformat.PackedRowIdMapping; import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.TextFieldMapper; -import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.nio.file.Path; @@ -38,7 +36,7 @@ * Verifies that documents are reordered according to the sort permutation * from the primary data format (Parquet). */ -public class LuceneWriterSortedFlushTests extends OpenSearchTestCase { +public class LuceneWriterSortedFlushTests extends LucenePluginBaseTests { private LuceneDataFormat dataFormat; @@ -48,10 +46,6 @@ public void setUp() throws Exception { dataFormat = new LuceneDataFormat(); } - private MappedFieldType mockTextField(String name) { - return new TextFieldMapper.TextFieldType(name); - } - /** * Writes 5 docs with row IDs 0..4, then flushes with a reverse permutation * (0→4, 1→3, 2→2, 3→1, 4→0). Verifies that the resulting Lucene segment diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java index 96f9b306fb097..96f49aa17e0f6 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java @@ -9,9 +9,7 @@ package org.opensearch.be.lucene.index; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -25,16 +23,14 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.NIOFSDirectory; import org.opensearch.be.lucene.LuceneDataFormat; +import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DeleteInput; import org.opensearch.index.engine.dataformat.FileInfos; import org.opensearch.index.engine.dataformat.FlushInput; import org.opensearch.index.engine.dataformat.WriteResult; import org.opensearch.index.engine.dataformat.Writer; import org.opensearch.index.engine.exec.WriterFileSet; -import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.TextFieldMapper; -import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.nio.file.Path; @@ -49,7 +45,7 @@ * Tests for {@link LuceneWriter} — the per-generation Lucene writer that creates * segments in isolated temp directories with force-merge to 1 segment on flush. */ -public class LuceneWriterTests extends OpenSearchTestCase { +public class LuceneWriterTests extends LucenePluginBaseTests { private LuceneDataFormat dataFormat; @@ -59,20 +55,6 @@ public void setUp() throws Exception { dataFormat = new LuceneDataFormat(); } - private MappedFieldType mockTextField(String name) { - return new TextFieldMapper.TextFieldType(name); - } - - private MappedFieldType mockKeywordField(String name) { - final FieldType keywordFieldType = new FieldType(); - keywordFieldType.setTokenized(false); - keywordFieldType.setStored(false); - keywordFieldType.setOmitNorms(true); - keywordFieldType.setIndexOptions(IndexOptions.DOCS); - keywordFieldType.freeze(); - return new KeywordFieldMapper.KeywordFieldType(name, keywordFieldType); - } - public void testAddDocAndFlushProducesSingleSegment() throws IOException { Path baseDir = createTempDir(); try ( @@ -236,6 +218,8 @@ public void testUnsupportedFieldTypeIsSilentlySkipped() throws IOException { MappedFieldType numericField = mock(MappedFieldType.class); when(numericField.typeName()).thenReturn("integer"); when(numericField.name()).thenReturn("count"); + // Empty capability map → no format owns this field; should be silently skipped + when(numericField.getCapabilityMap()).thenReturn(java.util.Map.of()); try ( LuceneWriter writer = new LuceneWriter( @@ -257,6 +241,39 @@ public void testUnsupportedFieldTypeIsSilentlySkipped() throws IOException { } } + public void testFieldOwnedByAnotherFormatIsSilentlySkipped() throws IOException { + Path baseDir = createTempDir(); + DataFormat otherFormat = mock(DataFormat.class); + when(otherFormat.name()).thenReturn("parquet"); + + MappedFieldType fieldOwnedByOther = mock(MappedFieldType.class); + when(fieldOwnedByOther.typeName()).thenReturn("integer"); + when(fieldOwnedByOther.name()).thenReturn("count"); + when(fieldOwnedByOther.getCapabilityMap()).thenReturn( + java.util.Map.of( + otherFormat, + java.util.Set.of(org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE) + ) + ); + + try ( + LuceneWriter writer = new LuceneWriter( + 1L, + 0L, + dataFormat, + baseDir, + null, + Codec.getDefault(), + null, + ConcurrentHashMap.newKeySet() + ) + ) { + LuceneDocumentInput input = new LuceneDocumentInput(); + input.addField(fieldOwnedByOther, 42); + assertEquals(0, input.getFinalInput().getFields().size()); + } + } + public void testMixedTextAndKeywordFields() throws IOException { Path baseDir = createTempDir(); MappedFieldType textField = mockTextField("title"); diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractCompositeEngineIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractCompositeEngineIT.java index f8f3292e55907..f14500e520faa 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractCompositeEngineIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractCompositeEngineIT.java @@ -82,11 +82,16 @@ protected void createCompositeIndex(String indexName, boolean withLuceneSecondar settingsBuilder.putList("index.composite.secondary_data_formats"); } + String keywordMapping = "type=keyword"; + if (false == withLuceneSecondary) { + keywordMapping += ",index=false"; + } + client().admin() .indices() .prepareCreate(indexName) .setSettings(settingsBuilder) - .setMapping("name", "type=keyword", "value", "type=integer") + .setMapping("name", keywordMapping, "value", "type=integer") .get(); ensureGreen(indexName); } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractSortedRefreshIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractSortedRefreshIT.java new file mode 100644 index 0000000000000..31bb35a38ae3a --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/AbstractSortedRefreshIT.java @@ -0,0 +1,538 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.opensearch.action.admin.indices.flush.FlushResponse; +import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexService; +import org.opensearch.index.engine.CommitStats; +import org.opensearch.index.engine.dataformat.DocumentInput; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.DataformatAwareCatalogSnapshot; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.parquet.bridge.ParquetFileMetadata; +import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class AbstractSortedRefreshIT extends OpenSearchIntegTestCase { + + protected static final String INDEX_NAME = "test-composite-refresh-sorted"; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .build(); + } + + @Override + public void tearDown() throws Exception { + try { + client().admin().indices().prepareDelete(INDEX_NAME).get(); + } catch (Exception e) { + // index may not exist + } + super.tearDown(); + } + + protected void createIndex(Settings settings) { + client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).setMapping(getMapping()).get(); + ensureGreen(INDEX_NAME); + } + + protected String[] getMapping() { + return new String[] { "name", "type=keyword", "age", "type=integer", "tag", "type=keyword" }; + } + + protected void indexDoc(String name, int age) { + IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name, "age", age).get(); + assertEquals(RestStatus.CREATED, response.status()); + } + + protected void indexDoc(String name, int age, String tag) { + IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name, "age", age, "tag", tag).get(); + assertEquals(RestStatus.CREATED, response.status()); + } + + protected void indexDocNullAge(String name) { + IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name).get(); + assertEquals(RestStatus.CREATED, response.status()); + } + + protected void flushAndRefresh() { + RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(INDEX_NAME).get(); + assertEquals(RestStatus.OK, refreshResponse.getStatus()); + FlushResponse flushResponse = client().admin().indices().prepareFlush(INDEX_NAME).setForce(true).get(); + assertEquals(RestStatus.OK, flushResponse.getStatus()); + } + + // ══════════════════════════════════════════════════════════════════════ + // Helpers: verification + // ══════════════════════════════════════════════════════════════════════ + + protected void verifyParquetRowCount(DataformatAwareCatalogSnapshot snapshot, int expectedTotalDocs) throws IOException { + Path parquetDir = getParquetDir(); + long totalRows = 0; + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + assertNotNull("Segment should have parquet files", wfs); + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + assertTrue("Parquet file should exist: " + filePath, Files.exists(filePath)); + ParquetFileMetadata metadata = RustBridge.getFileMetadata(filePath.toString()); + totalRows += metadata.numRows(); + } + } + assertEquals("Total rows should match ingested docs", expectedTotalDocs, totalRows); + } + + @SuppressForbidden(reason = "JSON parsing for sort order verification") + protected void verifyParquetSortOrder(DataformatAwareCatalogSnapshot snapshot) throws Exception { + Path parquetDir = getParquetDir(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + String json = RustBridge.readAsJson(filePath.toString()); + List> rows; + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + rows = parser.list().stream().map(o -> { + @SuppressWarnings("unchecked") + Map m = (Map) o; + return m; + }).toList(); + } + if (rows.size() <= 1) continue; + + for (int i = 1; i < rows.size(); i++) { + Object prevAge = rows.get(i - 1).get("age"); + Object currAge = rows.get(i).get("age"); + + // nulls first for age + if (prevAge == null && currAge == null) continue; + if (prevAge == null) continue; + if (currAge == null) { + fail("age null should come before non-null at row " + i); + } + + int prevAgeVal = ((Number) prevAge).intValue(); + int currAgeVal = ((Number) currAge).intValue(); + + assertTrue( + "age should be DESC but found " + prevAgeVal + " before " + currAgeVal + " at row " + i + " in " + file, + prevAgeVal >= currAgeVal + ); + + // When age is equal, verify name ASC (nulls last) + if (prevAgeVal == currAgeVal) { + Object prevName = rows.get(i - 1).get("name"); + Object currName = rows.get(i).get("name"); + + if (prevName != null && currName == null) continue; + if (prevName == null && currName != null) { + fail("name nulls should be last at row " + i + " in " + file); + } + if (prevName != null && currName != null) { + assertTrue( + "name should be ASC but found '" + prevName + "' before '" + currName + "' at row " + i + " in " + file, + ((String) prevName).compareTo((String) currName) <= 0 + ); + } + } + } + } + } + } + + protected void verifyLuceneDocCount(int expectedTotalDocs) throws IOException { + Path luceneDir = getLuceneDir(); + assertTrue("Lucene directory should exist", Files.exists(luceneDir)); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals("Lucene doc count should match", expectedTotalDocs, reader.numDocs()); + } + } + + /** + * Verifies Parquet sort order for a dynamic set of integer sort fields. + * Supports any combination of ASC/DESC with configurable null placement. + */ + @SuppressForbidden(reason = "JSON parsing for sort order verification") + protected void verifyParquetSortOrderMultiField( + DataformatAwareCatalogSnapshot snapshot, + String[] fieldNames, + String[] sortOrders, + String[] sortMissing + ) throws Exception { + Path parquetDir = getParquetDir(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + String json = RustBridge.readAsJson(filePath.toString()); + List> rows; + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + rows = parser.list().stream().map(o -> { + @SuppressWarnings("unchecked") + Map m = (Map) o; + return m; + }).toList(); + } + if (rows.size() <= 1) continue; + + for (int i = 1; i < rows.size(); i++) { + int cmp = compareRowsByMultiField(rows.get(i - 1), rows.get(i), fieldNames, sortOrders, sortMissing); + assertTrue( + "Sort order violated at row " + i + " in " + file + ": prev=" + rows.get(i - 1) + " curr=" + rows.get(i), + cmp <= 0 + ); + } + } + } + } + + /** + * Compares two rows by the multi-field sort key. Returns negative if prev comes before curr + * (correct order), zero if equal, positive if prev comes after curr (violation). + */ + protected int compareRowsByMultiField( + Map prev, + Map curr, + String[] fieldNames, + String[] sortOrders, + String[] sortMissing + ) { + for (int f = 0; f < fieldNames.length; f++) { + Object prevVal = prev.get(fieldNames[f]); + Object currVal = curr.get(fieldNames[f]); + boolean isAsc = "asc".equals(sortOrders[f]); + boolean nullsFirst = "_first".equals(sortMissing[f]); + + int cmp = compareValues(prevVal, currVal, isAsc, nullsFirst); + if (cmp != 0) return cmp; + } + return 0; + } + + /** + * Compares two nullable integer values according to sort direction and null placement. + * Returns negative if in correct order, zero if equal, positive if violated. + */ + protected int compareValues(Object prev, Object curr, boolean isAsc, boolean nullsFirst) { + if (prev == null && curr == null) return 0; + if (prev == null) { + // prev is null: correct if nullsFirst, violation if nullsLast + return nullsFirst ? -1 : 1; + } + if (curr == null) { + // curr is null: violation if nullsFirst, correct if nullsLast + return nullsFirst ? 1 : -1; + } + int prevInt = ((Number) prev).intValue(); + int currInt = ((Number) curr).intValue(); + int natural = Integer.compare(prevInt, currInt); + return isAsc ? natural : -natural; + } + + protected void verifyLuceneRowIdSequential() throws IOException { + Path luceneDir = getLuceneDir(); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + for (LeafReaderContext ctx : reader.leaves()) { + SortedNumericDocValues rowIdDV = ctx.reader().getSortedNumericDocValues(DocumentInput.ROW_ID_FIELD); + if (rowIdDV == null) continue; + + long expectedRowId = 0; + for (int doc = 0; doc < ctx.reader().maxDoc(); doc++) { + if (rowIdDV.advanceExact(doc)) { + long rowId = rowIdDV.nextValue(); + assertEquals( + DocumentInput.ROW_ID_FIELD + + " should be sequential, expected " + + expectedRowId + + " but got " + + rowId + + " at doc " + + doc, + expectedRowId, + rowId + ); + expectedRowId++; + } + } + } + } + } + + /** + * Verifies that the Lucene segment is physically sorted by the configured IndexSort + * (age DESC nulls first). Reads age from sorted numeric doc values. + */ + protected void verifyLuceneSortOrder() throws IOException { + Path luceneDir = getLuceneDir(); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + for (LeafReaderContext ctx : reader.leaves()) { + org.apache.lucene.index.LeafReader leaf = ctx.reader(); + int maxDoc = leaf.maxDoc(); + if (maxDoc <= 1) continue; + + SortedNumericDocValues ageDV = leaf.getSortedNumericDocValues("age"); + + Long prevAge = null; + boolean prevAgeNull = true; + for (int doc = 0; doc < maxDoc; doc++) { + Long age = null; + if (ageDV != null && ageDV.advanceExact(doc)) { + age = ageDV.nextValue(); + } + + if (doc > 0) { + // age DESC nulls first + if (prevAgeNull && age == null) { + // both null — ok + } else if (prevAgeNull) { + // prev was null, current is non-null — ok (nulls first) + } else if (age == null) { + fail("null age at doc " + doc + " came after non-null (expected nulls first)"); + } else { + assertTrue("age should be DESC at doc " + doc + ", got " + prevAge + " before " + age, prevAge >= age); + } + } + prevAge = age; + prevAgeNull = (age == null); + } + } + } + } + + @SuppressForbidden(reason = "JSON parsing for row ID verification") + protected void verifyParquetRowIdSequential(DataformatAwareCatalogSnapshot snapshot) throws Exception { + Path parquetDir = getParquetDir(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + String json = RustBridge.readAsJson(filePath.toString()); + List> rows; + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + rows = parser.list().stream().map(o -> { + @SuppressWarnings("unchecked") + Map m = (Map) o; + return m; + }).toList(); + } + + long expectedRowId = 0; + for (Map row : rows) { + Object rowIdObj = row.get(DocumentInput.ROW_ID_FIELD); + assertNotNull( + DocumentInput.ROW_ID_FIELD + " should be present in parquet row " + expectedRowId + " in " + file, + rowIdObj + ); + long rowId = ((Number) rowIdObj).longValue(); + assertEquals( + DocumentInput.ROW_ID_FIELD + + " should be sequential, expected " + + expectedRowId + + " but got " + + rowId + + " in " + + file, + expectedRowId, + rowId + ); + expectedRowId++; + } + } + } + } + + /** + * Reads Parquet rows and Lucene documents in physical (storage) order from the + * single-shard, single-segment index and asserts that at every position i, + * Parquet row i and Lucene doc i agree on keyword fields ({@code name} and + * {@code tag}). + *

+ * The Lucene secondary writer only persists text/keyword fields (in the inverted + * index) and {@code __row_id__} (as doc values). Numeric fields like {@code age} + * are intentionally NOT stored in the Lucene secondary — they live only in + * Parquet. Keyword fields are also stored only in the inverted index (no doc + * values), so we read them via TermsEnum/postings and reconstruct the per-doc + * value. + *

+ * The {@code tag} field is NOT part of the sort key — verifying it confirms the + * row ID rewrite correctly co-locates non-sort fields across formats too. + */ + @SuppressForbidden(reason = "JSON parsing for cross-format row alignment") + protected void verifyParquetAndLuceneRowsAlignedSequentially(DataformatAwareCatalogSnapshot snapshot) throws Exception { + // Read Parquet rows in physical order + Path parquetDir = getParquetDir(); + List> parquetRows = new ArrayList<>(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + for (String file : wfs.files()) { + String json = RustBridge.readAsJson(parquetDir.resolve(file).toString()); + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + for (Object obj : parser.list()) { + @SuppressWarnings("unchecked") + Map row = (Map) obj; + parquetRows.add(row); + } + } + } + } + + // Reconstruct per-doc keyword field values from Lucene's inverted index + Path luceneDir = getLuceneDir(); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals("Test assumes a single Lucene leaf for sequential alignment", 1, reader.leaves().size()); + org.apache.lucene.index.LeafReader leaf = reader.leaves().get(0).reader(); + + String[] luceneNames = readKeywordValuesPerDoc(leaf, "name"); + String[] luceneTags = readKeywordValuesPerDoc(leaf, "tag"); + + assertEquals("Parquet and Lucene must have same row count", parquetRows.size(), luceneNames.length); + + for (int i = 0; i < parquetRows.size(); i++) { + Map pq = parquetRows.get(i); + String pqName = (String) pq.get("name"); + String pqTag = (String) pq.get("tag"); + assertEquals("name mismatch at position " + i, pqName, luceneNames[i]); + assertEquals("tag mismatch at position " + i, pqTag, luceneTags[i]); + } + } + } + + /** + * Asserts that the {@code name} keyword field, read in Lucene doc order from + * the single-segment index, matches the given expected sequence. Used by + * unsorted-path tests to confirm insertion order is preserved end-to-end. + */ + protected void verifyLuceneNamesInOrder(String[] expectedNames) throws IOException { + Path luceneDir = getLuceneDir(); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals("Test assumes a single Lucene leaf for sequential alignment", 1, reader.leaves().size()); + org.apache.lucene.index.LeafReader leaf = reader.leaves().get(0).reader(); + String[] actual = readKeywordValuesPerDoc(leaf, "name"); + assertEquals("Lucene doc count should match expected sequence length", expectedNames.length, actual.length); + for (int i = 0; i < expectedNames.length; i++) { + assertEquals("name at lucene docId " + i, expectedNames[i], actual[i]); + } + } + } + + /** + * Reconstructs the per-doc keyword value for the given field by iterating + * the inverted index. Each doc is expected to have at most one term for the + * field. Returns an array indexed by Lucene doc ID. + */ + protected String[] readKeywordValuesPerDoc(org.apache.lucene.index.LeafReader leaf, String fieldName) throws IOException { + String[] values = new String[leaf.maxDoc()]; + org.apache.lucene.index.Terms terms = leaf.terms(fieldName); + assertNotNull("Lucene index should have field '" + fieldName + "' in inverted index", terms); + org.apache.lucene.index.TermsEnum it = terms.iterator(); + org.apache.lucene.util.BytesRef term; + while ((term = it.next()) != null) { + String value = term.utf8ToString(); + org.apache.lucene.index.PostingsEnum postings = it.postings(null); + int doc; + while ((doc = postings.nextDoc()) != org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS) { + values[doc] = value; + } + } + return values; + } + + // ══════════════════════════════════════════════════════════════════════ + // Helpers: accessors + // ══════════════════════════════════════════════════════════════════════ + + protected DataformatAwareCatalogSnapshot getCatalogSnapshot() throws IOException { + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, getNodeName()); + IndexService indexService = indicesService.indexServiceSafe(resolveIndex(INDEX_NAME)); + IndexShard shard = indexService.getShard(0); + CommitStats commitStats = shard.commitStats(); + assertNotNull(commitStats); + assertNotNull(commitStats.getUserData()); + assertTrue(commitStats.getUserData().containsKey(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY)); + return DataformatAwareCatalogSnapshot.deserializeFromString( + commitStats.getUserData().get(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY), + Function.identity() + ); + } + + protected Path getParquetDir() { + IndexShard shard = getPrimaryShard(); + return shard.shardPath().getDataPath().resolve("parquet"); + } + + protected Path getLuceneDir() { + IndexShard shard = getPrimaryShard(); + return shard.shardPath().resolveIndex(); + } + + protected IndexShard getPrimaryShard() { + String nodeName = getNodeName(); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName); + IndexService indexService = indicesService.indexServiceSafe(resolveIndex(INDEX_NAME)); + return indexService.getShard(0); + } + + protected String getNodeName() { + String nodeId = getClusterState().routingTable().index(INDEX_NAME).shard(0).primaryShard().currentNodeId(); + return getClusterState().nodes().get(nodeId).getName(); + } +} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeCommitDeletionIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeCommitDeletionIT.java index 80316bae72849..e54c10e506b0f 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeCommitDeletionIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeCommitDeletionIT.java @@ -81,9 +81,9 @@ private void createCompositeIndex() { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", "lucene") ) - .setMapping("field", "type=keyword") + .setMapping("field", "type=integer") .get(); ensureGreen(INDEX_NAME); } @@ -92,7 +92,7 @@ private void indexDocs(int count, int startId) { for (int i = startId; i < startId + count; i++) { assertEquals( RestStatus.CREATED, - client().prepareIndex().setIndex(INDEX_NAME).setId(String.valueOf(i)).setSource("field", "value_" + i).get().status() + client().prepareIndex().setIndex(INDEX_NAME).setId(String.valueOf(i)).setSource("field", i).get().status() ); } } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeConcurrentIndexingIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeConcurrentIndexingIT.java index 9ac7f3124a04c..828c77d2d2364 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeConcurrentIndexingIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeConcurrentIndexingIT.java @@ -1063,7 +1063,7 @@ private Settings indexSettings(int numShards) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", "lucene") .build(); } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDataCoherenceIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDataCoherenceIT.java new file mode 100644 index 0000000000000..b1f52729773cf --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDataCoherenceIT.java @@ -0,0 +1,546 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.opensearch.action.bulk.BulkRequestBuilder; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.engine.DataFormatAwareEngine; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicInteger; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class CompositeDataCoherenceIT extends OpenSearchIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + ParquetDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .build(); + } + + // ────────────────────────────────────────────────────────────────────────── + // Contract tests: InternalEngine equivalents for Parquet+Lucene composite + // ────────────────────────────────────────────────────────────────────────── + + /** + * Concurrent indexing must not lose docs. Both formats must have identical row counts. + * Reference: InternalEngineTests.testAppendConcurrently + */ + public void testConcurrentAppendsCrossFormatConsistency() throws Exception { + String indexName = "test-concurrent-appends"; + createParquetLuceneIndex(indexName); + + int numThreads = 4; + int docsPerThread = 25; + CyclicBarrier barrier = new CyclicBarrier(numThreads); + AtomicInteger successCount = new AtomicInteger(); + + Thread[] threads = new Thread[numThreads]; + for (int t = 0; t < numThreads; t++) { + final int threadId = t; + threads[t] = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < docsPerThread; i++) { + IndexResponse resp = client().prepareIndex(indexName) + .setSource("field_keyword", "t" + threadId + "_d" + i, "field_number", i) + .get(); + if (resp.status() == RestStatus.CREATED) successCount.incrementAndGet(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + threads[t].start(); + } + for (Thread t : threads) { + t.join(); + } + + int totalExpected = numThreads * docsPerThread; + assertEquals("all must succeed", totalExpected, successCount.get()); + + flushIndex(indexName); + assertParquetLuceneRowCountsMatch(indexName, totalExpected); + } + + /** + * Bulk indexing must be per-doc atomic — all docs visible, formats consistent. + * Reference: InternalEngineTests.testConcurrentWritesAndCommits + */ + public void testBulkIndexCrossFormatConsistency() throws Exception { + String indexName = "test-bulk-index"; + createParquetLuceneIndex(indexName); + + int batchSize = 50; + BulkRequestBuilder bulk = client().prepareBulk(); + for (int i = 0; i < batchSize; i++) { + bulk.add(client().prepareIndex(indexName).setSource("field_keyword", "bulk_" + i, "field_number", i)); + } + BulkResponse resp = bulk.get(); + assertFalse("no failures in bulk", resp.hasFailures()); + + flushIndex(indexName); + assertParquetLuceneRowCountsMatch(indexName, batchSize); + } + + /** + * Multiple flush cycles produce additive, consistent state across formats. + * Reference: InternalEngineTests.testShouldPeriodicallyFlush + */ + public void testMultipleFlushCyclesAdditive() throws Exception { + String indexName = "test-multi-flush"; + createParquetLuceneIndex(indexName); + + indexDocsTo(indexName, 5); + flushIndex(indexName); + assertParquetLuceneRowCountsMatch(indexName, 5); + + indexDocsTo(indexName, 7); + flushIndex(indexName); + assertParquetLuceneRowCountsMatch(indexName, 12); + + indexDocsTo(indexName, 3); + flushIndex(indexName); + assertParquetLuceneRowCountsMatch(indexName, 15); + } + + /** + * Refresh must make all buffered docs visible atomically across both formats. + * After a single refresh, both parquet and lucene must show the same new segment + * with the same row count — no partial visibility where one format lags. + * Reference: InternalEngineTests.testRefreshScopedSearcher + */ + public void testRefreshAtomicityAcrossFormats() throws Exception { + String indexName = "test-refresh-atomicity"; + createParquetLuceneIndex(indexName); + + indexDocsTo(indexName, 10); + + // Single refresh — not flush. Docs should become visible in both formats simultaneously. + DataFormatAwareEngine engine = getEngine(indexName); + engine.refresh("test"); + + try (GatedCloseable ref = engine.acquireSnapshot()) { + CatalogSnapshot snapshot = ref.get(); + long parquetRows = snapshot.getSearchableFiles("parquet").stream().mapToLong(WriterFileSet::numRows).sum(); + long luceneRows = snapshot.getSearchableFiles("lucene").stream().mapToLong(WriterFileSet::numRows).sum(); + assertEquals("refresh must make same docs visible in both formats", parquetRows, luceneRows); + assertEquals("all 10 docs must be visible after refresh", 10, parquetRows); + } + } + + /** + * flushAndClose must commit all buffered data. After reopen, both formats must have + * all docs that were indexed before shutdown. + * Reference: InternalEngineTests.testFlushAndClose + */ + public void testFlushAndClosePreservesAllData() throws Exception { + String indexName = "test-flush-and-close"; + createParquetLuceneIndex(indexName); + + indexDocsTo(indexName, 10); + // Do NOT flush explicitly — flushAndClose should handle it + + // flushAndClose via index close (which triggers flushAndClose on the engine) + client().admin().indices().prepareClose(indexName).get(); + client().admin().indices().prepareOpen(indexName).get(); + ensureGreen(indexName); + + DataFormatAwareEngine recovered = getEngine(indexName); + assertEquals("all 10 ops recovered via translog", 9, recovered.getSeqNoStats(-1).getMaxSeqNo()); + + // Flush + refresh to make recovered data visible in catalog + flushIndex(indexName); + recovered.refresh("verify"); + + long parquetRows = getRowCount(indexName, "parquet"); + long luceneRows = getRowCount(indexName, "lucene"); + assertEquals("post-close parquet and lucene must match", parquetRows, luceneRows); + assertEquals("all 10 docs must survive flushAndClose", 10, parquetRows); + } + + /** + * Translog replay after clean shutdown (close/reopen) must produce identical state + * in both formats. Index docs, flush some, then index more WITHOUT flushing. Close/reopen. + * The close triggers flushAndClose which persists uncommitted data via translog. + * After reopen, both formats must have all docs. + * Reference: InternalEngineTests.testTranslogReplay + testRecoverFromLocalTranslog + */ + public void testTranslogReplayCleanShutdownCrossFormatConsistency() throws Exception { + String indexName = "test-translog-replay-clean"; + createParquetLuceneIndex(indexName); + + // Phase 1: committed data + indexDocsTo(indexName, 8); + flushIndex(indexName); + + // Phase 2: uncommitted data (in translog only) + indexDocsTo(indexName, 5); + // NO explicit flush — close will trigger flushAndClose, translog holds these ops + + DataFormatAwareEngine engine = getEngine(indexName); + assertEquals("13 ops total", 12, engine.getSeqNoStats(-1).getMaxSeqNo()); + + // Close/reopen — close triggers flushAndClose, reopen replays from translog if needed + client().admin().indices().prepareClose(indexName).get(); + client().admin().indices().prepareOpen(indexName).get(); + ensureGreen(indexName); + + DataFormatAwareEngine recovered = getEngine(indexName); + assertEquals("all 13 ops recovered", 12, recovered.getSeqNoStats(-1).getMaxSeqNo()); + + flushIndex(indexName); + recovered.refresh("verify"); + + long parquetRows = getRowCount(indexName, "parquet"); + long luceneRows = getRowCount(indexName, "lucene"); + assertEquals("post-replay: formats must match", parquetRows, luceneRows); + assertEquals("post-replay: all 13 docs present", 13, parquetRows); + } + + /** + * Translog replay after unclean shutdown (failEngine) must produce identical state + * in both formats. Index docs, flush some, then index more WITHOUT flushing. + * Fail engine (simulates crash). After recovery, both formats must have all docs + * (committed + replayed from translog). + * Reference: InternalEngineTests.testTranslogReplay + */ + public void testTranslogReplayUncleanShutdownCrossFormatConsistency() throws Exception { + String indexName = "test-translog-replay-crash"; + createParquetLuceneIndex(indexName); + + // Phase 1: committed data + indexDocsTo(indexName, 8); + flushIndex(indexName); + + // Phase 2: uncommitted data (in translog only) + indexDocsTo(indexName, 5); + // NO flush — these are only in the translog + + DataFormatAwareEngine engine = getEngine(indexName); + assertEquals("13 ops total", 12, engine.getSeqNoStats(-1).getMaxSeqNo()); + + // Simulate unclean shutdown — engine fails without flushing + engine.failEngine("simulated-crash", new java.io.IOException("disk error")); + + // Reopen — triggers translog replay of the 5 uncommitted docs + client().admin().indices().prepareClose(indexName).get(); + client().admin().indices().prepareOpen(indexName).get(); + ensureGreen(indexName); + + DataFormatAwareEngine recovered = getEngine(indexName); + assertEquals("all 13 ops recovered", 12, recovered.getSeqNoStats(-1).getMaxSeqNo()); + + flushIndex(indexName); + recovered.refresh("verify"); + + long parquetRows = getRowCount(indexName, "parquet"); + long luceneRows = getRowCount(indexName, "lucene"); + assertEquals("post-replay: formats must match", parquetRows, luceneRows); + assertEquals("post-replay: all 13 docs present", 13, parquetRows); + } + + /** + * Concurrent indexing + refresh must produce consistent cross-format snapshots. + * While indexing threads are active, refresh threads take snapshots. Every snapshot + * must have matching row counts across both formats. + * Reference: InternalEngineTests.testConcurrentAppendUpdateAndRefresh + */ + public void testConcurrentIndexAndRefreshConsistency() throws Exception { + String indexName = "test-concurrent-refresh"; + createParquetLuceneIndex(indexName); + + int numIndexThreads = 3; + int docsPerThread = 20; + int numRefreshes = 10; + CyclicBarrier barrier = new CyclicBarrier(numIndexThreads + 1); + AtomicInteger successCount = new AtomicInteger(); + AtomicInteger inconsistencyCount = new AtomicInteger(); + + // Indexing threads + Thread[] indexThreads = new Thread[numIndexThreads]; + for (int t = 0; t < numIndexThreads; t++) { + final int threadId = t; + indexThreads[t] = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < docsPerThread; i++) { + IndexResponse resp = client().prepareIndex(indexName) + .setSource("field_keyword", "t" + threadId + "_d" + i, "field_number", i) + .get(); + if (resp.status() == RestStatus.CREATED) successCount.incrementAndGet(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + indexThreads[t].start(); + } + + // Refresh thread — checks cross-format consistency at each refresh point + Thread refreshThread = new Thread(() -> { + try { + barrier.await(); + DataFormatAwareEngine eng = getEngine(indexName); + for (int r = 0; r < numRefreshes; r++) { + eng.refresh("concurrent-check-" + r); + try (GatedCloseable ref = eng.acquireSnapshot()) { + CatalogSnapshot snap = ref.get(); + long pq = snap.getSearchableFiles("parquet").stream().mapToLong(WriterFileSet::numRows).sum(); + long lc = snap.getSearchableFiles("lucene").stream().mapToLong(WriterFileSet::numRows).sum(); + if (pq != lc) { + inconsistencyCount.incrementAndGet(); + } + } + Thread.sleep(5); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + refreshThread.start(); + + for (Thread t : indexThreads) { + t.join(); + } + refreshThread.join(); + + assertEquals("every intermediate refresh must show consistent cross-format counts", 0, inconsistencyCount.get()); + + // Final verification + flushIndex(indexName); + assertParquetLuceneRowCountsMatch(indexName, numIndexThreads * docsPerThread); + } + + /** + * After multiple refresh cycles, each segment must have matching numRows in both + * its parquet and lucene WriterFileSets. This verifies that row IDs are + * correctly assigned within each writer generation across formats. + * (No direct InternalEngine analog — unique to multi-format) + */ + public void testSegmentGenerationAlignmentAcrossFormats() throws Exception { + String indexName = "test-segment-alignment"; + createParquetLuceneIndex(indexName); + + // Create 3 segments with different sizes + indexDocsTo(indexName, 4); + flushIndex(indexName); + + indexDocsTo(indexName, 7); + flushIndex(indexName); + + indexDocsTo(indexName, 2); + flushIndex(indexName); + + DataFormatAwareEngine engine = getEngine(indexName); + engine.refresh("verify"); + + try (GatedCloseable ref = engine.acquireSnapshot()) { + CatalogSnapshot snapshot = ref.get(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet parquetWfs = segment.dfGroupedSearchableFiles().get("parquet"); + WriterFileSet luceneWfs = segment.dfGroupedSearchableFiles().get("lucene"); + + assertNotNull("segment gen=" + segment.generation() + " must have parquet files", parquetWfs); + assertNotNull("segment gen=" + segment.generation() + " must have lucene files", luceneWfs); + assertEquals( + "segment gen=" + segment.generation() + " must have same numRows in both formats", + parquetWfs.numRows(), + luceneWfs.numRows() + ); + assertTrue("segment gen=" + segment.generation() + " must have positive numRows", parquetWfs.numRows() > 0); + } + } + } + + /** + * Writer generation counter must resume from the committed value after recovery, + * not restart from 1. Otherwise new segments collide with committed segments in + * the CatalogSnapshot. + * Reference: InternalEngineTests.testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary + */ + public void testWriterGenerationResumesAfterRestart() throws Exception { + String indexName = "test-writer-gen-resume"; + createParquetLuceneIndex(indexName); + + // Create 3 segments — writer generations 1, 2, 3 + indexDocsTo(indexName, 5); + flushIndex(indexName); + indexDocsTo(indexName, 5); + flushIndex(indexName); + indexDocsTo(indexName, 5); + flushIndex(indexName); + + // Close and reopen + client().admin().indices().prepareClose(indexName).get(); + client().admin().indices().prepareOpen(indexName).get(); + ensureGreen(indexName); + + // Index more after reopen — should NOT collide with existing segment generations + indexDocsTo(indexName, 5); + flushIndex(indexName); + + DataFormatAwareEngine engine = getEngine(indexName); + engine.refresh("verify"); + + // Verify: 20 total docs, no generation conflicts + try (GatedCloseable ref = engine.acquireSnapshot()) { + CatalogSnapshot snapshot = ref.get(); + long totalParquet = snapshot.getSearchableFiles("parquet").stream().mapToLong(WriterFileSet::numRows).sum(); + long totalLucene = snapshot.getSearchableFiles("lucene").stream().mapToLong(WriterFileSet::numRows).sum(); + assertEquals("all 20 docs must be present in parquet", 20, totalParquet); + assertEquals("all 20 docs must be present in lucene", 20, totalLucene); + + // Verify no duplicate generations + java.util.List generations = snapshot.getSegments() + .stream() + .map(Segment::generation) + .collect(java.util.stream.Collectors.toList()); + assertEquals( + "all segment generations must be unique (no collision after restart)", + generations.size(), + generations.stream().distinct().count() + ); + } + } + + /** + * Concurrent indexing + flush must not lose documents. All docs that got + * CREATED responses must be visible after flush settles. + * Reference: InternalEngineTests.testConcurrentWritesAndCommits + */ + public void testConcurrentIndexAndFlushNoDataLoss() throws Exception { + String indexName = "test-concurrent-flush"; + createParquetLuceneIndex(indexName); + + int numIndexThreads = 3; + int docsPerThread = 30; + CyclicBarrier barrier = new CyclicBarrier(numIndexThreads + 1); + AtomicInteger successCount = new AtomicInteger(); + + Thread[] indexThreads = new Thread[numIndexThreads]; + for (int t = 0; t < numIndexThreads; t++) { + final int tid = t; + indexThreads[t] = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < docsPerThread; i++) { + IndexResponse resp = client().prepareIndex(indexName) + .setSource("field_keyword", "t" + tid + "_d" + i, "field_number", i) + .get(); + if (resp.status() == RestStatus.CREATED) successCount.incrementAndGet(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + indexThreads[t].start(); + } + + // Flush thread — flushes while indexing is in progress + Thread flushThread = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < 5; i++) { + Thread.sleep(10); + flushIndex(indexName); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + flushThread.start(); + + for (Thread t : indexThreads) { + t.join(); + } + flushThread.join(); + + // Final flush to commit any remaining buffered data + flushIndex(indexName); + DataFormatAwareEngine engine = getEngine(indexName); + engine.refresh("verify"); + + int expected = successCount.get(); + long parquetRows = getRowCount(indexName, "parquet"); + long luceneRows = getRowCount(indexName, "lucene"); + assertEquals("parquet must have all successful docs", expected, parquetRows); + assertEquals("lucene must have all successful docs", expected, luceneRows); + } + + // ── Helpers for Parquet+Lucene tests ── + + private void createParquetLuceneIndex(String indexName) { + CompositeEngineHelper.createCompositeIndexWithMapping(this, indexName, "parquet", Settings.EMPTY, "lucene"); + } + + private void indexDocsTo(String indexName, int count) { + CompositeEngineHelper.indexDocs(this, indexName, count); + } + + private void flushIndex(String indexName) { + CompositeEngineHelper.flush(this, indexName); + } + + private DataFormatAwareEngine getEngine(String indexName) { + return CompositeEngineHelper.getEngine(clusterService(), internalCluster(), indexName); + } + + private long getRowCount(String indexName, String formatName) throws IOException { + return CompositeEngineHelper.getRowCount(clusterService(), internalCluster(), indexName, formatName); + } + + private void assertParquetLuceneRowCountsMatch(String indexName, long expected) throws IOException { + DataFormatAwareEngine engine = getEngine(indexName); + engine.refresh("test"); + assertEquals("parquet row count", expected, getRowCount(indexName, "parquet")); + assertEquals("lucene row count", expected, getRowCount(indexName, "lucene")); + + CompositeEngineHelper.assertPerSegmentRowCountsMatch(engine, "parquet", "lucene"); + + // Checkpoint must be consistent with seqNo + long maxSeq = engine.getSeqNoStats(-1).getMaxSeqNo(); + long processedCp = engine.getProcessedLocalCheckpoint(); + assertEquals("processed checkpoint must equal maxSeqNo", maxSeq, processedCp); + } +} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDynamicMappingIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDynamicMappingIT.java index 4315976ef865b..d59fab0091b09 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDynamicMappingIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeDynamicMappingIT.java @@ -20,9 +20,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.action.admin.indices.flush.FlushResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.arrow.allocator.ArrowBasePlugin; import org.opensearch.be.datafusion.DataFusionPlugin; @@ -98,114 +96,6 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } - /** - * Tests that documents with dynamically added fields are indexed successfully - * into a composite parquet index. The flow is: - * 1. Create index with initial mapping (field_keyword, field_number) - * 2. Index documents matching the initial schema - * 3. Index documents with NEW fields not in the original mapping (dynamic mapping) - * 4. Refresh + flush - * 5. Verify all documents indexed, parquet files generated with correct segments - */ - public void testDynamicMappingWithParquet() throws Exception { - Settings indexSettings = parquetOnlySettings(); - - // Create index with initial mapping - CreateIndexResponse createResponse = client().admin() - .indices() - .prepareCreate(INDEX_NAME) - .setSettings(indexSettings) - .setMapping("field_keyword", "type=keyword", "field_number", "type=integer") - .get(); - assertTrue("Index creation should be acknowledged", createResponse.isAcknowledged()); - ensureGreen(INDEX_NAME); - - // Index documents with initial schema - for (int i = 0; i < 5; i++) { - IndexResponse response = client().prepareIndex() - .setIndex(INDEX_NAME) - .setSource("field_keyword", "value_" + i, "field_number", i) - .get(); - assertEquals(RestStatus.CREATED, response.status()); - } - - // Verify dynamic fields are NOT yet in the mapping - GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings(INDEX_NAME).get(); - Map mappingSource = mappingsResponse.mappings().get(INDEX_NAME).sourceAsMap(); - @SuppressWarnings("unchecked") - Map properties = (Map) mappingSource.get("properties"); - assertTrue("Mapping should contain initial field 'field_keyword'", properties.containsKey("field_keyword")); - assertTrue("Mapping should contain initial field 'field_number'", properties.containsKey("field_number")); - assertFalse("Mapping should NOT contain 'dynamic_text' yet", properties.containsKey("dynamic_text")); - assertFalse("Mapping should NOT contain 'dynamic_long' yet", properties.containsKey("dynamic_long")); - - // Index documents with NEW dynamic fields - indexDocsWithDynamicFields(INDEX_NAME, 5, 10); - - // Verify dynamic fields are now present in the mapping. - // Note: The cluster-manager applies its own cluster state after publication completes, - // so there's a brief window where GetMappings on the cluster-manager may return stale data. - assertMappingsContain(INDEX_NAME, "dynamic_text", "dynamic_long"); - - // Refresh and flush to produce parquet files - RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(INDEX_NAME).get(); - assertEquals(RestStatus.OK, refreshResponse.getStatus()); - FlushResponse flushResponse = client().admin().indices().prepareFlush(INDEX_NAME).get(); - assertEquals(RestStatus.OK, flushResponse.getStatus()); - - // Verify parquet files on disk - IndexShard shard = getIndexShard(INDEX_NAME); - Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); - assertTrue("Parquet directory should exist", Files.isDirectory(parquetDir)); - - try (GatedCloseable> parquetFilesRef = listParquetFiles(parquetDir, shard)) { - List parquetFiles = parquetFilesRef.get(); - assertFalse("Should have at least one parquet file", parquetFiles.isEmpty()); - - // Verify row count from parquet file metadata - long totalRows = getParquetRowCount(parquetFiles); - assertEquals("Total rows across parquet files should equal 10", 10, totalRows); - - // Verify content via readAsJson - List> allRows = readAllParquetRows(parquetFiles); - assertEquals(10, allRows.size()); - // Verify dynamic fields are present in the rows that should have them - assertDynamicFieldCount(allRows, "dynamic_text", 5); - } - - // After flush, the writer is now immutable. Index more docs with another new dynamic field - // to verify schema evolution across writer generations (new writer created with fresh schema). - for (int i = 10; i < 15; i++) { - IndexResponse response = client().prepareIndex() - .setIndex(INDEX_NAME) - .setSource("field_keyword", "value_" + i, "field_number", i, "dynamic_extra", "extra_" + i) - .get(); - assertEquals(RestStatus.CREATED, response.status()); - } - - // Verify new dynamic field in mapping - assertMappingsContain(INDEX_NAME, "dynamic_extra"); - - // Refresh + flush again - client().admin().indices().prepareRefresh(INDEX_NAME).get(); - client().admin().indices().prepareFlush(INDEX_NAME).get(); - - // Verify all 15 rows on disk - try (GatedCloseable> parquetFilesRef = listParquetFiles(parquetDir, shard)) { - List parquetFiles = parquetFilesRef.get(); - long totalRows = getParquetRowCount(parquetFiles); - assertEquals("Total rows across parquet files should equal 15", 15, totalRows); - - // Verify content - List> allRows = readAllParquetRows(parquetFiles); - assertEquals(15, allRows.size()); - assertDynamicFieldCount(allRows, "dynamic_extra", 5); - } - - ensureGreen(INDEX_NAME); - ensureNoActiveMerges(INDEX_NAME); - } - /** * Tests dynamic mapping with parquet primary + lucene secondary. * Verifies that dynamically added fields appear in both formats. @@ -263,14 +153,18 @@ public void testDynamicMappingWithParquetPrimaryLuceneSecondary() throws Excepti assertEquals("All 10 Lucene docs should have __row_id__", 10, rowsWithRowId); // Verify that the lucene index has the expected indexed fields (inverted index) - assertLuceneIndexedFieldsPresent(luceneDir, Set.of("field_keyword", "dynamic_text", "dynamic_text.keyword")); + assertLuceneIndexedFieldsPresent(luceneDir, Set.of("field_keyword", "dynamic_text")); ensureNoActiveMerges(indexName); } public void testConflictingDynamicMappings() { String indexName = "test-conflict"; - CreateIndexResponse createResponse = client().admin().indices().prepareCreate(indexName).setSettings(parquetOnlySettings()).get(); + CreateIndexResponse createResponse = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(parquetPrimaryLuceneSecondarySettings()) + .get(); assertTrue(createResponse.isAcknowledged()); ensureGreen(indexName); @@ -290,34 +184,6 @@ public void testConflictingDynamicMappings() { } } - public void testConcurrentDynamicUpdates() throws Throwable { - String indexName = "test-concurrent"; - - CreateIndexResponse createResponse = client().admin().indices().prepareCreate(indexName).setSettings(parquetOnlySettings()).get(); - assertTrue(createResponse.isAcknowledged()); - ensureGreen(indexName); - - final int numThreads = 32; - runConcurrentIndexing(indexName, numThreads); - - // Verify all 64 fields in mapping - assertConcurrentMappings(indexName, numThreads); - - // Verify parquet content - IndexShard shard = getIndexShard(indexName); - Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); - try (GatedCloseable> parquetFilesRef = listParquetFiles(parquetDir, shard)) { - List parquetFiles = parquetFilesRef.get(); - - assertEquals("Total rows should equal 64", 64, getParquetRowCount(parquetFiles)); - - List> allRows = readAllParquetRows(parquetFiles); - assertEquals(64, allRows.size()); - assertConcurrentFieldValues(allRows, numThreads); - } - ensureNoActiveMerges(indexName); - } - /** * Tests concurrent dynamic mapping updates with parquet primary + lucene secondary. * Verifies both formats contain all dynamically created fields. diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldCapabilityIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldCapabilityIT.java new file mode 100644 index 0000000000000..275b475c9bb2c --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldCapabilityIT.java @@ -0,0 +1,894 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Integration tests validating field capability assignment for composite indices. + * Verifies that supported field types create successfully, documents can be indexed, + * and both parquet and lucene structures contain expected fields. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class CompositeFieldCapabilityIT extends AbstractCompositeEngineIT { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).build(); + } + + private Settings dfaSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene") + .build(); + } + + private void startCluster() { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + } + + private void assertIndexCreationSucceeds(String indexName, String fieldName, String mappingType) { + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping(fieldName, mappingType) + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + } + + private void assertIndexCreationFails(String indexName, String fieldName, String mappingType) { + expectThrows( + MapperParsingException.class, + () -> client().admin().indices().prepareCreate(indexName).setSettings(dfaSettings()).setMapping(fieldName, mappingType).get() + ); + } + + // === SUPPORTED FIELDS (expect 200) === + + public void testLongFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-long", "field", "type=long"); + } + + public void testIntegerFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-integer", "field", "type=integer"); + } + + public void testShortFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-short", "field", "type=short"); + } + + public void testByteFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-byte", "field", "type=byte"); + } + + public void testDoubleFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-double", "field", "type=double"); + } + + public void testFloatFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-float", "field", "type=float"); + } + + public void testHalfFloatFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-half-float", "field", "type=half_float"); + } + + public void testUnsignedLongFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-unsigned-long", "field", "type=unsigned_long"); + } + + public void testDateFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-date", "field", "type=date"); + } + + public void testDateNanosFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-date-nanos", "field", "type=date_nanos"); + } + + public void testBooleanFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-boolean", "field", "type=boolean,index=false"); + } + + public void testKeywordFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-keyword", "field", "type=keyword"); + } + + public void testKeywordFieldSupportedWithIgnoreAbove() { + startCluster(); + assertIndexCreationSucceeds("test-keyword", "field", "type=keyword,ignore_above=256"); + } + + public void testTextFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-text", "field", "type=text"); + } + + public void testIpFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-ip", "field", "type=ip"); + } + + public void testBinaryFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-binary", "field", "type=binary,store=true"); + } + + public void testMatchOnlyTextFieldSupported() { + startCluster(); + assertIndexCreationSucceeds("test-match-only-text", "field", "type=match_only_text"); + } + + // === UNSUPPORTED FIELDS (expect 400 MapperParsingException) === + + public void testGeoPointFieldUnsupported() { + startCluster(); + assertIndexCreationFails("test-geo-point", "field", "type=geo_point"); + } + + public void testGeoShapeFieldUnsupported() { + startCluster(); + assertIndexCreationFails("test-geo-shape", "field", "type=geo_shape"); + } + + public void testCompletionFieldUnsupported() { + startCluster(); + assertIndexCreationFails("test-completion", "field", "type=completion"); + } + + public void testNestedFieldUnsupported() { + startCluster(); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> client().admin() + .indices() + .prepareCreate("test-nested") + .setSettings(dfaSettings()) + .setMapping("field", "type=nested") + .get() + ); + assertTrue(ex.getMessage().contains("nested type is not supported with pluggable data format")); + } + + public void testFlatObjectFieldUnsupported() { + startCluster(); + assertIndexCreationFails("test-flat-object", "field", "type=flat_object"); + } + + public void testWildcardFieldUnsupported() { + startCluster(); + assertIndexCreationFails("test-wildcard", "field", "type=wildcard"); + } + + // === SPECIAL CASES === + + public void testMultiFieldTextWithKeywordSubfield() throws Exception { + startCluster(); + String mapping = "{\n" + + " \"properties\": {\n" + + " \"field\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"raw\": { \"type\": \"keyword\" }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate("test-multi-field") + .setSettings(dfaSettings()) + .setMapping(mapping) + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen("test-multi-field"); + } + + public void testFieldWithOnlyDocValues() { + startCluster(); + assertIndexCreationSucceeds("test-doc-values-only", "field", "type=keyword,index=false,doc_values=true"); + } + + public void testFieldWithOnlyStored() { + startCluster(); + assertIndexCreationSucceeds("test-stored-only", "field", "type=keyword,index=false,doc_values=false,store=true"); + } + + public void testFieldWithAllDisabled() { + startCluster(); + assertIndexCreationFails("test-all-disabled", "field", "type=keyword,index=false,doc_values=false,store=false"); + } + + public void testMultipleFieldsAllSupported() { + startCluster(); + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate("test-multiple-fields") + .setSettings(dfaSettings()) + .setMapping("f_long", "type=long", "f_keyword", "type=keyword", "f_date", "type=date", "f_text", "type=text") + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen("test-multiple-fields"); + } + + // === INDEXING + STRUCTURE VERIFICATION TESTS === + + /** + * Tests indexing and dynamic mapping for ALL supported field types. + * Creates an index with every supported type, indexes documents, adds a dynamic field, + * and verifies parquet/lucene structures contain expected fields and all docs coexist. + */ + public void testAllSupportedFieldTypesIndexAndVerify() throws Exception { + startCluster(); + String indexName = "test-all-types-index"; + String mapping = "{\n" + + " \"properties\": {\n" + + " \"f_long\": { \"type\": \"long\" },\n" + + " \"f_integer\": { \"type\": \"integer\" },\n" + + " \"f_short\": { \"type\": \"short\" },\n" + + " \"f_byte\": { \"type\": \"byte\" },\n" + + " \"f_double\": { \"type\": \"double\" },\n" + + " \"f_float\": { \"type\": \"float\" },\n" + + " \"f_half_float\": { \"type\": \"half_float\" },\n" + + " \"f_unsigned_long\": { \"type\": \"unsigned_long\" },\n" + + " \"f_date\": { \"type\": \"date\" },\n" + + " \"f_date_nanos\": { \"type\": \"date_nanos\" },\n" + + " \"f_boolean\": { \"type\": \"boolean\", \"index\": false },\n" + + " \"f_keyword\": { \"type\": \"keyword\" },\n" + + " \"f_keyword_ignore\": { \"type\": \"keyword\", \"ignore_above\": 256 },\n" + + " \"f_text\": { \"type\": \"text\" },\n" + + " \"f_ip\": { \"type\": \"ip\" },\n" + + " \"f_match_only_text\": { \"type\": \"match_only_text\" }\n" + + " }\n" + + "}"; + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping(mapping) + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index doc with all field types populated + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("1") + .setSource( + "f_long", + 100L, + "f_integer", + 42, + "f_short", + 7, + "f_byte", + 1, + "f_double", + 3.14, + "f_float", + 2.5f, + "f_half_float", + 1.5f, + "f_unsigned_long", + 9999999999L, + "f_date", + "2024-01-15", + "f_date_nanos", + "2024-01-15T10:30:00.123456789Z", + "f_boolean", + true, + "f_keyword", + "alpha", + "f_keyword_ignore", + "short_val", + "f_text", + "hello world", + "f_ip", + "192.168.1.1", + "f_match_only_text", + "searchable text" + ) + .get() + .status() + ); + + // Index second doc with some fields + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("2") + .setSource( + "f_long", + 200L, + "f_integer", + 99, + "f_keyword", + "beta", + "f_text", + "second doc", + "f_boolean", + false, + "f_date", + "2024-06-01" + ) + .get() + .status() + ); + + // Dynamic mapping: add a new field not in original mapping + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("3") + .setSource("f_long", 300L, "f_keyword", "gamma", "f_text", "third with dynamic", "dynamic_new_field", "dynamic_value") + .get() + .status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify all 3 docs exist + assertDocCount(indexName, 3); + + // Verify parquet contains expected fields + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(3, parquetRows.size()); + + // All original fields should appear in at least one row + assertTrue("f_long in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_long"))); + assertTrue("f_integer in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_integer"))); + assertTrue("f_short in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_short"))); + assertTrue("f_byte in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_byte"))); + assertTrue("f_double in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_double"))); + assertTrue("f_float in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_float"))); + assertTrue("f_half_float in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_half_float"))); + assertTrue("f_unsigned_long in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_unsigned_long"))); + assertTrue("f_date in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_date"))); + assertTrue("f_date_nanos in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_date_nanos"))); + assertTrue("f_boolean in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_boolean"))); + assertTrue("f_keyword in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_keyword"))); + assertTrue("f_keyword_ignore in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_keyword_ignore"))); + assertTrue("f_text in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_text"))); + assertTrue("f_ip in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_ip"))); + assertTrue("f_match_only_text in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("f_match_only_text"))); + + // Dynamic field should appear + assertTrue("dynamic_new_field in parquet", parquetRows.stream().anyMatch(r -> r.containsKey("dynamic_new_field"))); + + // Verify lucene has expected indexed fields (inverted index: text/keyword types) + Path luceneDir = shard.shardPath().resolveIndex(); + Set luceneFields = getLuceneFields(luceneDir); + assertTrue("Lucene should have 'f_keyword'", luceneFields.contains("f_keyword")); + assertTrue("Lucene should have 'f_keyword_ignore'", luceneFields.contains("f_keyword_ignore")); + assertTrue("Lucene should have 'f_text'", luceneFields.contains("f_text")); + assertTrue("Lucene should have 'f_match_only_text'", luceneFields.contains("f_match_only_text")); + assertTrue("Lucene should have 'dynamic_new_field'", luceneFields.contains("dynamic_new_field")); + } + + /** + * Tests that a keyword field with ignore_above can be indexed and the sourceKeywordFieldType + * is stored in parquet when the value exceeds ignore_above. + * Also verifies dynamic mapping works and old+new docs coexist. + */ + public void testKeywordIgnoreAboveIndexAndVerify() throws Exception { + startCluster(); + String indexName = "test-keyword-ignore-above-verify"; + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping("field", "type=keyword,ignore_above=10", "id_field", "type=keyword") + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index doc with value within ignore_above + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("1").setSource("field", "short", "id_field", "doc1").get().status() + ); + // Index doc with value exceeding ignore_above (value ignored, sourceKeywordFieldType stores raw) + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("2") + .setSource("field", "this_exceeds_ignore_above_limit", "id_field", "doc2") + .get() + .status() + ); + + // Dynamic mapping: add a new field + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("3") + .setSource("field", "dynamic", "id_field", "doc3", "new_dynamic_field", "hello") + .get() + .status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify doc count via stats + assertDocCount(indexName, 3); + + // Verify parquet contains expected fields + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(3, parquetRows.size()); + + // id_field should be in all rows + assertEquals(3, parquetRows.stream().filter(r -> r.containsKey("id_field")).count()); + + // Verify ignore_above behavior: + // - doc1 ("short", len=5 <= 10): field should have the normalized value + // - doc2 ("this_exceeds_ignore_above_limit", len=32 > 10): field should be null/absent, + // but _ignored_source.field should store the raw text value + Map doc1Row = parquetRows.stream().filter(r -> "doc1".equals(r.get("id_field"))).findFirst().orElseThrow(); + Map doc2Row = parquetRows.stream().filter(r -> "doc2".equals(r.get("id_field"))).findFirst().orElseThrow(); + + // doc1: value within ignore_above — field is stored, no source keyword field needed + assertEquals("short", doc1Row.get("field")); + + // doc2: value exceeds ignore_above — field is null (ignored), source keyword stores raw value + assertNull("field should be null for doc2 (exceeds ignore_above)", doc2Row.get("field")); + assertEquals( + "sourceKeywordFieldType should store raw value for doc2", + "this_exceeds_ignore_above_limit", + doc2Row.get("_ignored_source.field") + ); + + // new_dynamic_field should appear in parquet (at least 1 row has it) + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("new_dynamic_field"))); + + // Verify lucene has expected indexed fields + Path luceneDir = shard.shardPath().resolveIndex(); + Set luceneFields = getLuceneFields(luceneDir); + assertTrue("Lucene should have field 'field'", luceneFields.contains("field")); + assertTrue("Lucene should have field 'id_field'", luceneFields.contains("id_field")); + assertTrue("Lucene should have dynamic field 'new_dynamic_field'", luceneFields.contains("new_dynamic_field")); + } + + /** + * Tests that a keyword field with a normalizer stores the raw value in sourceKeywordFieldType + * even when the value is within ignore_above. The normalized value goes into the main field, + * and the original un-normalized value is stored separately for source derivation. + */ + public void testKeywordNormalizerStoresSourceSeparately() throws Exception { + startCluster(); + String indexName = "test-keyword-normalizer-verify"; + + Settings settings = Settings.builder() + .put(dfaSettings()) + .put("index.analysis.normalizer.my_lower.type", "custom") + .putList("index.analysis.normalizer.my_lower.filter", "lowercase") + .build(); + + String mapping = "{\n" + + " \"properties\": {\n" + + " \"field\": { \"type\": \"keyword\", \"normalizer\": \"my_lower\" },\n" + + " \"id_field\": { \"type\": \"keyword\" }\n" + + " }\n" + + "}"; + + CreateIndexResponse response = client().admin().indices().prepareCreate(indexName).setSettings(settings).setMapping(mapping).get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index doc — value is within any ignore_above but normalizer transforms it + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("1").setSource("field", "Hello", "id_field", "doc1").get().status() + ); + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("2").setSource("field", "WORLD", "id_field", "doc2").get().status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + assertDocCount(indexName, 2); + + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(2, parquetRows.size()); + + Map doc1Row = parquetRows.stream().filter(r -> "doc1".equals(r.get("id_field"))).findFirst().orElseThrow(); + Map doc2Row = parquetRows.stream().filter(r -> "doc2".equals(r.get("id_field"))).findFirst().orElseThrow(); + + // Main field stores the normalized (lowercased) value + assertEquals("hello", doc1Row.get("field")); + assertEquals("world", doc2Row.get("field")); + + // sourceKeywordFieldType stores the original un-normalized value for source derivation + assertEquals("Hello", doc1Row.get("_ignored_source.field")); + assertEquals("WORLD", doc2Row.get("_ignored_source.field")); + } + + /** + * Tests that a keyword field with a normalizer stores the raw value in sourceKeywordFieldType + * even when the value is within ignore_above. The normalized value goes into the main field, + * and the original un-normalized value is stored separately for source derivation. + */ + public void testKeywordNormalizerStoresSourceSeparatelyWhenDynamicFieldIsPresent() throws Exception { + startCluster(); + String indexName = "test-keyword-normalizer-verify-dynamic-field"; + + Settings settings = Settings.builder() + .put(dfaSettings()) + .put("index.analysis.normalizer.my_lower.type", "custom") + .putList("index.analysis.normalizer.my_lower.filter", "lowercase") + .build(); + + String mapping = "{\n" + + " \"properties\": {\n" + + " \"field\": { \"type\": \"keyword\", \"normalizer\": \"my_lower\" }\n" + + " }\n" + + "}"; + + CreateIndexResponse response = client().admin().indices().prepareCreate(indexName).setSettings(settings).setMapping(mapping).get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index doc — value is within any ignore_above but normalizer transforms it + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("1").setSource("field", "Hello", "id_field1", "doc1").get().status() + ); + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("2").setSource("field", "WORLD", "id_field2", "doc2").get().status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + assertDocCount(indexName, 2); + + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(2, parquetRows.size()); + + Map doc1Row = parquetRows.stream().filter(r -> "doc1".equals(r.get("id_field1"))).findFirst().orElseThrow(); + Map doc2Row = parquetRows.stream().filter(r -> "doc2".equals(r.get("id_field2"))).findFirst().orElseThrow(); + + // Main field stores the normalized (lowercased) value + assertEquals("hello", doc1Row.get("field")); + assertEquals("world", doc2Row.get("field")); + + // sourceKeywordFieldType stores the original un-normalized value for source derivation + assertEquals("Hello", doc1Row.get("_ignored_source.field")); + assertEquals("WORLD", doc2Row.get("_ignored_source.field")); + } + + /** + * Tests multi-field text+keyword indexing and verifies both parquet and lucene structures. + * Also adds a dynamic field and verifies old+new docs coexist. + */ + public void testMultiFieldIndexAndVerify() throws Exception { + startCluster(); + String indexName = "test-multi-field-verify"; + String mapping = "{\n" + + " \"properties\": {\n" + + " \"content\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"raw\": { \"type\": \"keyword\" }\n" + + " }\n" + + " },\n" + + " \"tag\": { \"type\": \"keyword\" }\n" + + " }\n" + + "}"; + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping(mapping) + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index initial docs + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("1").setSource("content", "hello world", "tag", "greeting").get().status() + ); + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("2").setSource("content", "foo bar", "tag", "test").get().status() + ); + + // Dynamic mapping: add new field + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("3").setSource("content", "dynamic doc", "tag", "new", "extra_field", 42).get().status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify doc count + assertDocCount(indexName, 3); + + // Verify parquet + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(3, parquetRows.size()); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("content"))); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("tag"))); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("extra_field"))); + + // Verify lucene has text field and keyword sub-field + Path luceneDir = shard.shardPath().resolveIndex(); + Set luceneFields = getLuceneFields(luceneDir); + assertTrue("Lucene should have 'content'", luceneFields.contains("content")); + assertTrue("Lucene should have 'content.raw'", luceneFields.contains("content.raw")); + assertTrue("Lucene should have 'tag'", luceneFields.contains("tag")); + } + + /** + * Tests that multiple supported field types can be indexed together and verified. + * Also adds a dynamic field and verifies old+new docs coexist. + */ + public void testMultipleFieldTypesIndexAndVerify() throws Exception { + startCluster(); + String indexName = "test-multi-types-verify"; + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping("f_long", "type=long", "f_keyword", "type=keyword", "f_date", "type=date", "f_text", "type=text") + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index initial docs + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("1") + .setSource("f_long", 100L, "f_keyword", "alpha", "f_date", "2024-01-01", "f_text", "some text") + .get() + .status() + ); + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("2") + .setSource("f_long", 200L, "f_keyword", "beta", "f_date", "2024-06-15", "f_text", "more text") + .get() + .status() + ); + + // Dynamic mapping: add new field + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId("3") + .setSource("f_long", 300L, "f_keyword", "gamma", "f_date", "2024-12-31", "f_text", "final", "dyn_bool", true) + .get() + .status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify doc count + assertDocCount(indexName, 3); + + // Verify parquet + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(3, parquetRows.size()); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("f_long"))); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("f_keyword"))); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("f_date"))); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("f_text"))); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("dyn_bool"))); + + // Verify lucene + Path luceneDir = shard.shardPath().resolveIndex(); + Set luceneFields = getLuceneFields(luceneDir); + assertTrue("Lucene should have 'f_keyword'", luceneFields.contains("f_keyword")); + assertTrue("Lucene should have 'f_text'", luceneFields.contains("f_text")); + } + + // === MAPPING UPDATE FAILURE TEST === + + /** + * Tests that index creation succeeds, a document can be added, but a mapping update + * with an unsupported field type causes the shard to fail when applying the mapping. + */ + public void testMappingUpdateFailsWithUnsupportedField() throws Exception { + startCluster(); + String indexName = "test-mapping-update-fail"; + assertIndexCreationSucceeds(indexName, "field", "type=keyword"); + + // Index a document successfully + assertEquals(RestStatus.CREATED, client().prepareIndex(indexName).setId("1").setSource("field", "value1").get().status()); + + client().admin().indices().prepareRefresh(indexName).get(); + + // Verify doc was indexed + assertDocCount(indexName, 1); + + // Attempt to update mapping with an unsupported field type (geo_point). + // The cluster-manager accepts the mapping update, but the data node fails + // when trying to apply it (capability assignment fails for geo_point). + client().admin() + .indices() + .preparePutMapping(indexName) + .setSource("{\"properties\":{\"unsupported_geo\":{\"type\":\"geo_point\"}}}", MediaTypeRegistry.JSON) + .get(); + + // The shard should become unhealthy as the mapping update fails on the data node. + // Wait for the cluster to detect the failure. + assertBusy(() -> { + String health = client().admin().cluster().prepareHealth(indexName).get().getStatus().name(); + assertTrue( + "Index should be RED or YELLOW after unsupported mapping update, got: " + health, + health.equals("RED") || health.equals("YELLOW") + ); + }); + } + + /** + * Tests dynamic mapping with a numeric field and verifies old+new docs coexist. + */ + public void testDynamicMappingNumericField() throws Exception { + startCluster(); + String indexName = "test-dynamic-mapping"; + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping("name", "type=keyword") + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index initial doc + assertEquals(RestStatus.CREATED, client().prepareIndex(indexName).setId("1").setSource("name", "first").get().status()); + + // Index doc with dynamic numeric field + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName).setId("2").setSource("name", "second", "dynamic_num", 42).get().status() + ); + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify both docs exist + assertDocCount(indexName, 2); + + // Verify parquet has both docs + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(2, parquetRows.size()); + assertTrue(parquetRows.stream().anyMatch(r -> r.containsKey("dynamic_num"))); + } + + // ══════════════════════════════════════════════════════════════════════ + // Private helpers + // ══════════════════════════════════════════════════════════════════════ + + private void assertDocCount(String indexName, long expectedCount) { + IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setDocs(true).get(); + assertEquals(expectedCount, stats.getIndex(indexName).getPrimaries().getDocs().getCount()); + } + + @SuppressForbidden(reason = "JSON parsing for test verification of parquet output") + private List> readAllParquetRows(Path parquetDir, IndexShard shard) throws IOException { + assertTrue("Parquet directory should exist", Files.isDirectory(parquetDir)); + List> allRows = new ArrayList<>(); + try (GatedCloseable snapshot = shard.getCatalogSnapshot()) { + for (Segment segment : snapshot.get().getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + if (wfs != null) { + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + allRows.addAll(parseJsonRows(RustBridge.readAsJson(filePath.toString()))); + } + } + } + } + return allRows; + } + + @SuppressWarnings("unchecked") + @SuppressForbidden(reason = "JSON parsing for test verification of parquet output") + private List> parseJsonRows(String json) throws IOException { + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + return parser.list().stream().map(o -> (Map) o).collect(Collectors.toList()); + } + } + + private Set getLuceneFields(Path luceneDir) throws IOException { + Set allFields = new HashSet<>(); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + for (LeafReaderContext ctx : reader.leaves()) { + for (FieldInfo fi : ctx.reader().getFieldInfos()) { + allFields.add(fi.name); + } + } + } + return allFields; + } +} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldConfigRandomizedIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldConfigRandomizedIT.java new file mode 100644 index 0000000000000..3a661122674d2 --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeFieldConfigRandomizedIT.java @@ -0,0 +1,345 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Randomized integration tests for composite field configurations. + * Verifies that fields with various index/doc_values combinations are correctly + * stored in parquet (columnar) and lucene (inverted index) based on their capabilities. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class CompositeFieldConfigRandomizedIT extends AbstractCompositeEngineIT { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).build(); + } + + private Settings dfaSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene") + .build(); + } + + private void startCluster() { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + } + + /** + * Tests randomized field configurations where keyword and numeric fields randomly + * have index=true or index=false. Verifies that: + * - All fields appear in parquet regardless of index setting + * - Only indexed fields appear in lucene FieldInfos + * - __row_id__ is always present in lucene + */ + public void testRandomizedFieldConfigIndexingAndVerification() throws Exception { + startCluster(); + String indexName = "test-randomized-field-config"; + + // Randomly decide index=true/false for keyword and numeric fields + boolean kw1Indexed = randomBoolean(); + boolean kw2Indexed = randomBoolean(); + boolean kw3Indexed = randomBoolean(); + boolean longIndexed = randomBoolean(); + boolean doubleIndexed = randomBoolean(); + + // Text fields always need index=true + logger.info( + "Random config: f_keyword_1.index={}, f_keyword_2.index={}, f_keyword_3.index={}, " + + "f_long.index={}, f_double.index={}, f_text_1.index=true, f_text_2.index=true, f_match_only_text.index=true", + kw1Indexed, + kw2Indexed, + kw3Indexed, + longIndexed, + doubleIndexed + ); + + String mapping = "{\n" + + " \"properties\": {\n" + + " \"f_keyword_1\": { \"type\": \"keyword\", \"index\": " + + kw1Indexed + + ", \"doc_values\": true },\n" + + " \"f_keyword_2\": { \"type\": \"keyword\", \"index\": " + + kw2Indexed + + ", \"doc_values\": true },\n" + + " \"f_keyword_3\": { \"type\": \"keyword\", \"index\": " + + kw3Indexed + + ", \"doc_values\": true },\n" + + " \"f_text_1\": { \"type\": \"text\" },\n" + + " \"f_text_2\": { \"type\": \"text\" },\n" + + " \"f_match_only_text\": { \"type\": \"match_only_text\" },\n" + + " \"f_long\": { \"type\": \"long\", \"index\": " + + longIndexed + + ", \"doc_values\": true },\n" + + " \"f_double\": { \"type\": \"double\", \"index\": " + + doubleIndexed + + ", \"doc_values\": true }\n" + + " }\n" + + "}"; + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping(mapping) + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index 5 documents with values for all fields + for (int i = 1; i <= 5; i++) { + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId(String.valueOf(i)) + .setSource( + "f_keyword_1", + "kw1_val_" + i, + "f_keyword_2", + "kw2_val_" + i, + "f_keyword_3", + "kw3_val_" + i, + "f_text_1", + "text one content " + i, + "f_text_2", + "text two content " + i, + "f_match_only_text", + "match only " + i, + "f_long", + (long) (i * 100), + "f_double", + i * 1.5 + ) + .get() + .status() + ); + } + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify doc count = 5 + assertDocCount(indexName, 5); + + // Verify parquet has columnar fields (keyword + numeric always have doc_values) + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(5, parquetRows.size()); + + // keyword and numeric fields always stored in parquet (COLUMNAR_STORAGE via doc_values) + for (String field : List.of("f_keyword_1", "f_keyword_2", "f_keyword_3", "f_long", "f_double")) { + String finalField = field; + assertTrue( + field + " should be present in parquet with non-null values", + parquetRows.stream().allMatch(r -> r.containsKey(finalField) && r.get(finalField) != null) + ); + } + // text/match_only_text are stored in parquet via STORED_FIELDS but may appear differently + for (String field : List.of("f_text_1", "f_text_2")) { + String finalField = field; + assertTrue(field + " should be present in parquet", parquetRows.stream().anyMatch(r -> r.containsKey(finalField))); + } + + // Verify lucene fields + Path luceneDir = shard.shardPath().resolveIndex(); + Set luceneFields = getLuceneFields(luceneDir); + + // __row_id__ is always present + assertTrue("__row_id__ should always be in lucene", luceneFields.contains("__row_id__")); + + // Text fields with index=true should be in lucene + assertTrue("f_text_1 should be in lucene", luceneFields.contains("f_text_1")); + assertTrue("f_text_2 should be in lucene", luceneFields.contains("f_text_2")); + assertTrue("f_match_only_text should be in lucene", luceneFields.contains("f_match_only_text")); + + // Keyword fields: present in lucene only if indexed + assertFieldInLucene(luceneFields, "f_keyword_1", kw1Indexed); + assertFieldInLucene(luceneFields, "f_keyword_2", kw2Indexed); + assertFieldInLucene(luceneFields, "f_keyword_3", kw3Indexed); + + // Numeric fields with index=false should NOT appear in lucene FieldInfos + if (!longIndexed) { + assertFalse("f_long with index=false should NOT be in lucene", luceneFields.contains("f_long")); + } + if (!doubleIndexed) { + assertFalse("f_double with index=false should NOT be in lucene", luceneFields.contains("f_double")); + } + } + + /** + * Tests keyword and numeric fields with index=false, doc_values=true (columnar-only). + * Verifies that parquet stores the data (COLUMNAR_STORAGE) while lucene does not + * have these fields (no FULL_TEXT_SEARCH requested). + */ + public void testColumnarOnlyFieldsIndexAndVerify() throws Exception { + startCluster(); + String indexName = "test-columnar-only"; + + String mapping = "{\n" + + " \"properties\": {\n" + + " \"col_keyword\": { \"type\": \"keyword\", \"index\": false, \"doc_values\": true },\n" + + " \"col_long\": { \"type\": \"long\", \"index\": false, \"doc_values\": true },\n" + + " \"col_double\": { \"type\": \"double\", \"index\": false, \"doc_values\": true }\n" + + " }\n" + + "}"; + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(dfaSettings()) + .setMapping(mapping) + .get(); + assertTrue(response.isAcknowledged()); + ensureGreen(indexName); + + // Index 5 documents + for (int i = 1; i <= 5; i++) { + assertEquals( + RestStatus.CREATED, + client().prepareIndex(indexName) + .setId(String.valueOf(i)) + .setSource("col_keyword", "value_" + i, "col_long", (long) (i * 10), "col_double", i * 2.5) + .get() + .status() + ); + } + + client().admin().indices().prepareRefresh(indexName).get(); + client().admin().indices().prepareFlush(indexName).get(); + + // Verify doc count + assertDocCount(indexName, 5); + + // Verify parquet has all columnar fields with data + IndexShard shard = getPrimaryShard(indexName); + Path parquetDir = shard.shardPath().getDataPath().resolve("parquet"); + List> parquetRows = readAllParquetRows(parquetDir, shard); + assertEquals(5, parquetRows.size()); + + for (String field : List.of("col_keyword", "col_long", "col_double")) { + String finalField = field; + assertTrue( + field + " should be in parquet (COLUMNAR_STORAGE satisfied)", + parquetRows.stream().allMatch(r -> r.containsKey(finalField) && r.get(finalField) != null) + ); + } + + // Verify lucene does NOT have these fields (no FULL_TEXT_SEARCH requested) + Path luceneDir = shard.shardPath().resolveIndex(); + Set luceneFields = getLuceneFields(luceneDir); + + assertFalse("col_keyword should NOT be in lucene (index=false)", luceneFields.contains("col_keyword")); + assertFalse("col_long should NOT be in lucene (index=false)", luceneFields.contains("col_long")); + assertFalse("col_double should NOT be in lucene (index=false)", luceneFields.contains("col_double")); + + // __row_id__ should still be present + assertTrue("__row_id__ should always be in lucene", luceneFields.contains("__row_id__")); + } + + // ══════════════════════════════════════════════════════════════════════ + // Private helpers + // ══════════════════════════════════════════════════════════════════════ + + private void assertFieldInLucene(Set luceneFields, String fieldName, boolean shouldBePresent) { + if (shouldBePresent) { + assertTrue(fieldName + " with index=true should be in lucene", luceneFields.contains(fieldName)); + } else { + assertFalse(fieldName + " with index=false should NOT be in lucene", luceneFields.contains(fieldName)); + } + } + + private void assertDocCount(String indexName, long expectedCount) { + IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setDocs(true).get(); + assertEquals(expectedCount, stats.getIndex(indexName).getPrimaries().getDocs().getCount()); + } + + @SuppressForbidden(reason = "JSON parsing for test verification") + private List> readAllParquetRows(Path parquetDir, IndexShard shard) throws IOException { + assertTrue("Parquet directory should exist", Files.isDirectory(parquetDir)); + List> allRows = new ArrayList<>(); + try (GatedCloseable snapshot = shard.getCatalogSnapshot()) { + for (Segment segment : snapshot.get().getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + if (wfs != null) { + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + allRows.addAll(parseJsonRows(RustBridge.readAsJson(filePath.toString()))); + } + } + } + } + return allRows; + } + + @SuppressWarnings("unchecked") + @SuppressForbidden(reason = "JSON parsing for test verification") + private List> parseJsonRows(String json) throws IOException { + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + return parser.list().stream().map(o -> (Map) o).collect(Collectors.toList()); + } + } + + private Set getLuceneFields(Path luceneDir) throws IOException { + Set allFields = new HashSet<>(); + try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { + for (LeafReaderContext ctx : reader.leaves()) { + for (FieldInfo fi : ctx.reader().getFieldInfos()) { + allFields.add(fi.name); + } + } + } + return allFields; + } +} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java index b1391f2c7840d..725f4b4d83d16 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java @@ -112,7 +112,7 @@ protected Settings nodeSettings(int nodeOrdinal) { // ══════════════════════════════════════════════════════════════════════ /** - * Verifies background merge produces a valid merged parquet file + * Verifies background merge produces a valid merged parquet file and lucene segment * with correct row count and source files cleaned up. */ public void testBackgroundMerge() throws Exception { @@ -130,39 +130,13 @@ public void testBackgroundMerge() throws Exception { int totalDocs = refreshCycles * docsPerCycle; DataformatAwareCatalogSnapshot snapshot = waitForMerge(refreshCycles); - assertEquals(Set.of("parquet"), snapshot.getDataFormats()); + assertEquals(Set.of("parquet", "lucene"), snapshot.getDataFormats()); verifyRowCount(snapshot, totalDocs); verifySegmentGenerationUniqueness(snapshot); verifyNoOrphanFiles(snapshot); } - /** - * Verifies sorted merge with age DESC (nulls first), name ASC (nulls last). - */ - public void testSortedMerge() throws Exception { - client().admin() - .indices() - .prepareCreate(INDEX_NAME) - .setSettings(sortedSettings()) - .setMapping("name", "type=keyword", "age", "type=integer") - .get(); - ensureGreen(INDEX_NAME); - - int docsPerCycle = 10; - int refreshCycles = 15; - indexDocsWithNullsAcrossRefreshes(refreshCycles, docsPerCycle); - int totalDocs = refreshCycles * docsPerCycle; - - DataformatAwareCatalogSnapshot snapshot = waitForMerge(refreshCycles); - assertEquals(Set.of("parquet"), snapshot.getDataFormats()); - - verifyRowCount(snapshot, totalDocs); - verifySortOrder(snapshot); - verifySegmentGenerationUniqueness(snapshot); - verifyNoOrphanFiles(snapshot); - } - /** * Verifies composite merge with Parquet as primary and Lucene as secondary: *

    @@ -366,22 +340,7 @@ private Settings unsortedSettings() { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") - .build(); - } - - private Settings sortedSettings() { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.refresh_interval", "-1") - .put("index.pluggable.dataformat.enabled", true) - .put("index.pluggable.dataformat", "composite") - .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") - .putList("index.sort.field", "age", "name") - .putList("index.sort.order", "desc", "asc") - .putList("index.sort.missing", "_first", "_last") + .putList("index.composite.secondary_data_formats", "lucene") .build(); } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquet3TierSettingsIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquet3TierSettingsIT.java index 80d1c62eb122d..670740c04a073 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquet3TierSettingsIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquet3TierSettingsIT.java @@ -382,7 +382,7 @@ private void createIndex(Settings extraIndexSettings) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", "lucene") .put(extraIndexSettings); client().admin() @@ -407,7 +407,7 @@ private void createAllTypeIndex(Settings extraIndexSettings) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", "lucene") .put(extraIndexSettings); client().admin() @@ -416,7 +416,7 @@ private void createAllTypeIndex(Settings extraIndexSettings) { .setSettings(builder) .setMapping( "col_utf8", - "type=keyword", + "type=keyword,index=false", "col_int32", "type=integer", "col_int64", @@ -426,9 +426,9 @@ private void createAllTypeIndex(Settings extraIndexSettings) { "col_float64", "type=double", "col_boolean", - "type=boolean", + "type=boolean,index=false", "col_binary", - "type=binary", + "type=binary,store=true", "col_timestamp", "type=date" ) diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java index 99e1622e818a4..e6b39ed05109e 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java @@ -16,24 +16,18 @@ import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; -import org.opensearch.action.bulk.BulkRequestBuilder; -import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.arrow.allocator.ArrowBasePlugin; import org.opensearch.be.datafusion.DataFusionPlugin; import org.opensearch.be.lucene.LucenePlugin; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.framework.ParquetOnlyDataFormatPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.CommitStats; -import org.opensearch.index.engine.DataFormatAwareEngine; -import org.opensearch.index.engine.exec.Segment; -import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; import org.opensearch.index.engine.exec.coord.DataformatAwareCatalogSnapshot; -import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -41,8 +35,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Set; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; /** @@ -66,7 +58,7 @@ protected Collection> nodePlugins() { // framework's ingest pool, so the framework plugin must be installed. return Arrays.asList( ArrowBasePlugin.class, - ParquetDataFormatPlugin.class, + ParquetOnlyDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class, DataFusionPlugin.class @@ -95,9 +87,7 @@ public void testCreateCompositeParquetIndex() throws IOException { .indices() .prepareCreate(INDEX_NAME) .setSettings(indexSettings) - .setMapping("field_text", "type=text") - .setMapping("field_keyword", "type=keyword") - .setMapping("field_number", "type=integer") + .setMapping("field_text", "type=text,index=false", "field_keyword", "type=keyword,index=false", "field_number", "type=integer") .get(); assertTrue("Index creation should be acknowledged", response.isAcknowledged()); @@ -180,7 +170,7 @@ public void testCompositeParquetWithLuceneSecondary() throws IOException { .indices() .prepareCreate(indexName) .setSettings(indexSettings) - .setMapping("field_text", "type=text", "field_keyword", "type=keyword", "field_number", "type=integer") + .setMapping("field_text", "type=text,index=false", "field_keyword", "type=keyword,index=false", "field_number", "type=integer") .get(); assertTrue("Index creation should be acknowledged", response.isAcknowledged()); @@ -251,493 +241,6 @@ public void testCompositeParquetWithLuceneSecondary() throws IOException { ensureGreen(indexName); } - // ────────────────────────────────────────────────────────────────────────── - // Contract tests: InternalEngine equivalents for Parquet+Lucene composite - // ────────────────────────────────────────────────────────────────────────── - - /** - * Concurrent indexing must not lose docs. Both formats must have identical row counts. - * Reference: InternalEngineTests.testAppendConcurrently - */ - public void testConcurrentAppendsCrossFormatConsistency() throws Exception { - String indexName = "test-concurrent-appends"; - createParquetLuceneIndex(indexName); - - int numThreads = 4; - int docsPerThread = 25; - CyclicBarrier barrier = new CyclicBarrier(numThreads); - AtomicInteger successCount = new AtomicInteger(); - - Thread[] threads = new Thread[numThreads]; - for (int t = 0; t < numThreads; t++) { - final int threadId = t; - threads[t] = new Thread(() -> { - try { - barrier.await(); - for (int i = 0; i < docsPerThread; i++) { - IndexResponse resp = client().prepareIndex(indexName) - .setSource("field_keyword", "t" + threadId + "_d" + i, "field_number", i) - .get(); - if (resp.status() == RestStatus.CREATED) successCount.incrementAndGet(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - threads[t].start(); - } - for (Thread t : threads) { - t.join(); - } - - int totalExpected = numThreads * docsPerThread; - assertEquals("all must succeed", totalExpected, successCount.get()); - - flushIndex(indexName); - assertParquetLuceneRowCountsMatch(indexName, totalExpected); - } - - /** - * Bulk indexing must be per-doc atomic — all docs visible, formats consistent. - * Reference: InternalEngineTests.testConcurrentWritesAndCommits - */ - public void testBulkIndexCrossFormatConsistency() throws Exception { - String indexName = "test-bulk-index"; - createParquetLuceneIndex(indexName); - - int batchSize = 50; - BulkRequestBuilder bulk = client().prepareBulk(); - for (int i = 0; i < batchSize; i++) { - bulk.add(client().prepareIndex(indexName).setSource("field_keyword", "bulk_" + i, "field_number", i)); - } - BulkResponse resp = bulk.get(); - assertFalse("no failures in bulk", resp.hasFailures()); - - flushIndex(indexName); - assertParquetLuceneRowCountsMatch(indexName, batchSize); - } - - /** - * Multiple flush cycles produce additive, consistent state across formats. - * Reference: InternalEngineTests.testShouldPeriodicallyFlush - */ - public void testMultipleFlushCyclesAdditive() throws Exception { - String indexName = "test-multi-flush"; - createParquetLuceneIndex(indexName); - - indexDocsTo(indexName, 5); - flushIndex(indexName); - assertParquetLuceneRowCountsMatch(indexName, 5); - - indexDocsTo(indexName, 7); - flushIndex(indexName); - assertParquetLuceneRowCountsMatch(indexName, 12); - - indexDocsTo(indexName, 3); - flushIndex(indexName); - assertParquetLuceneRowCountsMatch(indexName, 15); - } - - /** - * Refresh must make all buffered docs visible atomically across both formats. - * After a single refresh, both parquet and lucene must show the same new segment - * with the same row count — no partial visibility where one format lags. - * Reference: InternalEngineTests.testRefreshScopedSearcher - */ - public void testRefreshAtomicityAcrossFormats() throws Exception { - String indexName = "test-refresh-atomicity"; - createParquetLuceneIndex(indexName); - - indexDocsTo(indexName, 10); - - // Single refresh — not flush. Docs should become visible in both formats simultaneously. - DataFormatAwareEngine engine = getEngine(indexName); - engine.refresh("test"); - - try (GatedCloseable ref = engine.acquireSnapshot()) { - CatalogSnapshot snapshot = ref.get(); - long parquetRows = snapshot.getSearchableFiles("parquet").stream().mapToLong(WriterFileSet::numRows).sum(); - long luceneRows = snapshot.getSearchableFiles("lucene").stream().mapToLong(WriterFileSet::numRows).sum(); - assertEquals("refresh must make same docs visible in both formats", parquetRows, luceneRows); - assertEquals("all 10 docs must be visible after refresh", 10, parquetRows); - } - } - - /** - * flushAndClose must commit all buffered data. After reopen, both formats must have - * all docs that were indexed before shutdown. - * Reference: InternalEngineTests.testFlushAndClose - */ - public void testFlushAndClosePreservesAllData() throws Exception { - String indexName = "test-flush-and-close"; - createParquetLuceneIndex(indexName); - - indexDocsTo(indexName, 10); - // Do NOT flush explicitly — flushAndClose should handle it - - // flushAndClose via index close (which triggers flushAndClose on the engine) - client().admin().indices().prepareClose(indexName).get(); - client().admin().indices().prepareOpen(indexName).get(); - ensureGreen(indexName); - - DataFormatAwareEngine recovered = getEngine(indexName); - assertEquals("all 10 ops recovered via translog", 9, recovered.getSeqNoStats(-1).getMaxSeqNo()); - - // Flush + refresh to make recovered data visible in catalog - flushIndex(indexName); - recovered.refresh("verify"); - - long parquetRows = getRowCount(indexName, "parquet"); - long luceneRows = getRowCount(indexName, "lucene"); - assertEquals("post-close parquet and lucene must match", parquetRows, luceneRows); - assertEquals("all 10 docs must survive flushAndClose", 10, parquetRows); - } - - /** - * Translog replay after clean shutdown (close/reopen) must produce identical state - * in both formats. Index docs, flush some, then index more WITHOUT flushing. Close/reopen. - * The close triggers flushAndClose which persists uncommitted data via translog. - * After reopen, both formats must have all docs. - * Reference: InternalEngineTests.testTranslogReplay + testRecoverFromLocalTranslog - */ - public void testTranslogReplayCleanShutdownCrossFormatConsistency() throws Exception { - String indexName = "test-translog-replay-clean"; - createParquetLuceneIndex(indexName); - - // Phase 1: committed data - indexDocsTo(indexName, 8); - flushIndex(indexName); - - // Phase 2: uncommitted data (in translog only) - indexDocsTo(indexName, 5); - // NO explicit flush — close will trigger flushAndClose, translog holds these ops - - DataFormatAwareEngine engine = getEngine(indexName); - assertEquals("13 ops total", 12, engine.getSeqNoStats(-1).getMaxSeqNo()); - - // Close/reopen — close triggers flushAndClose, reopen replays from translog if needed - client().admin().indices().prepareClose(indexName).get(); - client().admin().indices().prepareOpen(indexName).get(); - ensureGreen(indexName); - - DataFormatAwareEngine recovered = getEngine(indexName); - assertEquals("all 13 ops recovered", 12, recovered.getSeqNoStats(-1).getMaxSeqNo()); - - flushIndex(indexName); - recovered.refresh("verify"); - - long parquetRows = getRowCount(indexName, "parquet"); - long luceneRows = getRowCount(indexName, "lucene"); - assertEquals("post-replay: formats must match", parquetRows, luceneRows); - assertEquals("post-replay: all 13 docs present", 13, parquetRows); - } - - /** - * Translog replay after unclean shutdown (failEngine) must produce identical state - * in both formats. Index docs, flush some, then index more WITHOUT flushing. - * Fail engine (simulates crash). After recovery, both formats must have all docs - * (committed + replayed from translog). - * Reference: InternalEngineTests.testTranslogReplay - */ - public void testTranslogReplayUncleanShutdownCrossFormatConsistency() throws Exception { - String indexName = "test-translog-replay-crash"; - createParquetLuceneIndex(indexName); - - // Phase 1: committed data - indexDocsTo(indexName, 8); - flushIndex(indexName); - - // Phase 2: uncommitted data (in translog only) - indexDocsTo(indexName, 5); - // NO flush — these are only in the translog - - DataFormatAwareEngine engine = getEngine(indexName); - assertEquals("13 ops total", 12, engine.getSeqNoStats(-1).getMaxSeqNo()); - - // Simulate unclean shutdown — engine fails without flushing - engine.failEngine("simulated-crash", new java.io.IOException("disk error")); - - // Reopen — triggers translog replay of the 5 uncommitted docs - client().admin().indices().prepareClose(indexName).get(); - client().admin().indices().prepareOpen(indexName).get(); - ensureGreen(indexName); - - DataFormatAwareEngine recovered = getEngine(indexName); - assertEquals("all 13 ops recovered", 12, recovered.getSeqNoStats(-1).getMaxSeqNo()); - - flushIndex(indexName); - recovered.refresh("verify"); - - long parquetRows = getRowCount(indexName, "parquet"); - long luceneRows = getRowCount(indexName, "lucene"); - assertEquals("post-replay: formats must match", parquetRows, luceneRows); - assertEquals("post-replay: all 13 docs present", 13, parquetRows); - } - - /** - * Concurrent indexing + refresh must produce consistent cross-format snapshots. - * While indexing threads are active, refresh threads take snapshots. Every snapshot - * must have matching row counts across both formats. - * Reference: InternalEngineTests.testConcurrentAppendUpdateAndRefresh - */ - public void testConcurrentIndexAndRefreshConsistency() throws Exception { - String indexName = "test-concurrent-refresh"; - createParquetLuceneIndex(indexName); - - int numIndexThreads = 3; - int docsPerThread = 20; - int numRefreshes = 10; - CyclicBarrier barrier = new CyclicBarrier(numIndexThreads + 1); - AtomicInteger successCount = new AtomicInteger(); - AtomicInteger inconsistencyCount = new AtomicInteger(); - - // Indexing threads - Thread[] indexThreads = new Thread[numIndexThreads]; - for (int t = 0; t < numIndexThreads; t++) { - final int threadId = t; - indexThreads[t] = new Thread(() -> { - try { - barrier.await(); - for (int i = 0; i < docsPerThread; i++) { - IndexResponse resp = client().prepareIndex(indexName) - .setSource("field_keyword", "t" + threadId + "_d" + i, "field_number", i) - .get(); - if (resp.status() == RestStatus.CREATED) successCount.incrementAndGet(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - indexThreads[t].start(); - } - - // Refresh thread — checks cross-format consistency at each refresh point - Thread refreshThread = new Thread(() -> { - try { - barrier.await(); - DataFormatAwareEngine eng = getEngine(indexName); - for (int r = 0; r < numRefreshes; r++) { - eng.refresh("concurrent-check-" + r); - try (GatedCloseable ref = eng.acquireSnapshot()) { - CatalogSnapshot snap = ref.get(); - long pq = snap.getSearchableFiles("parquet").stream().mapToLong(WriterFileSet::numRows).sum(); - long lc = snap.getSearchableFiles("lucene").stream().mapToLong(WriterFileSet::numRows).sum(); - if (pq != lc) { - inconsistencyCount.incrementAndGet(); - } - } - Thread.sleep(5); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - refreshThread.start(); - - for (Thread t : indexThreads) { - t.join(); - } - refreshThread.join(); - - assertEquals("every intermediate refresh must show consistent cross-format counts", 0, inconsistencyCount.get()); - - // Final verification - flushIndex(indexName); - assertParquetLuceneRowCountsMatch(indexName, numIndexThreads * docsPerThread); - } - - /** - * After multiple refresh cycles, each segment must have matching numRows in both - * its parquet and lucene WriterFileSets. This verifies that row IDs are - * correctly assigned within each writer generation across formats. - * (No direct InternalEngine analog — unique to multi-format) - */ - public void testSegmentGenerationAlignmentAcrossFormats() throws Exception { - String indexName = "test-segment-alignment"; - createParquetLuceneIndex(indexName); - - // Create 3 segments with different sizes - indexDocsTo(indexName, 4); - flushIndex(indexName); - - indexDocsTo(indexName, 7); - flushIndex(indexName); - - indexDocsTo(indexName, 2); - flushIndex(indexName); - - DataFormatAwareEngine engine = getEngine(indexName); - engine.refresh("verify"); - - try (GatedCloseable ref = engine.acquireSnapshot()) { - CatalogSnapshot snapshot = ref.get(); - for (Segment segment : snapshot.getSegments()) { - WriterFileSet parquetWfs = segment.dfGroupedSearchableFiles().get("parquet"); - WriterFileSet luceneWfs = segment.dfGroupedSearchableFiles().get("lucene"); - - assertNotNull("segment gen=" + segment.generation() + " must have parquet files", parquetWfs); - assertNotNull("segment gen=" + segment.generation() + " must have lucene files", luceneWfs); - assertEquals( - "segment gen=" + segment.generation() + " must have same numRows in both formats", - parquetWfs.numRows(), - luceneWfs.numRows() - ); - assertTrue("segment gen=" + segment.generation() + " must have positive numRows", parquetWfs.numRows() > 0); - } - } - } - - /** - * Writer generation counter must resume from the committed value after recovery, - * not restart from 1. Otherwise new segments collide with committed segments in - * the CatalogSnapshot. - * Reference: InternalEngineTests.testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary - */ - public void testWriterGenerationResumesAfterRestart() throws Exception { - String indexName = "test-writer-gen-resume"; - createParquetLuceneIndex(indexName); - - // Create 3 segments — writer generations 1, 2, 3 - indexDocsTo(indexName, 5); - flushIndex(indexName); - indexDocsTo(indexName, 5); - flushIndex(indexName); - indexDocsTo(indexName, 5); - flushIndex(indexName); - - // Close and reopen - client().admin().indices().prepareClose(indexName).get(); - client().admin().indices().prepareOpen(indexName).get(); - ensureGreen(indexName); - - // Index more after reopen — should NOT collide with existing segment generations - indexDocsTo(indexName, 5); - flushIndex(indexName); - - DataFormatAwareEngine engine = getEngine(indexName); - engine.refresh("verify"); - - // Verify: 20 total docs, no generation conflicts - try (GatedCloseable ref = engine.acquireSnapshot()) { - CatalogSnapshot snapshot = ref.get(); - long totalParquet = snapshot.getSearchableFiles("parquet").stream().mapToLong(WriterFileSet::numRows).sum(); - long totalLucene = snapshot.getSearchableFiles("lucene").stream().mapToLong(WriterFileSet::numRows).sum(); - assertEquals("all 20 docs must be present in parquet", 20, totalParquet); - assertEquals("all 20 docs must be present in lucene", 20, totalLucene); - - // Verify no duplicate generations - java.util.List generations = snapshot.getSegments() - .stream() - .map(Segment::generation) - .collect(java.util.stream.Collectors.toList()); - assertEquals( - "all segment generations must be unique (no collision after restart)", - generations.size(), - generations.stream().distinct().count() - ); - } - } - - /** - * Concurrent indexing + flush must not lose documents. All docs that got - * CREATED responses must be visible after flush settles. - * Reference: InternalEngineTests.testConcurrentWritesAndCommits - */ - public void testConcurrentIndexAndFlushNoDataLoss() throws Exception { - String indexName = "test-concurrent-flush"; - createParquetLuceneIndex(indexName); - - int numIndexThreads = 3; - int docsPerThread = 30; - CyclicBarrier barrier = new CyclicBarrier(numIndexThreads + 1); - AtomicInteger successCount = new AtomicInteger(); - - Thread[] indexThreads = new Thread[numIndexThreads]; - for (int t = 0; t < numIndexThreads; t++) { - final int tid = t; - indexThreads[t] = new Thread(() -> { - try { - barrier.await(); - for (int i = 0; i < docsPerThread; i++) { - IndexResponse resp = client().prepareIndex(indexName) - .setSource("field_keyword", "t" + tid + "_d" + i, "field_number", i) - .get(); - if (resp.status() == RestStatus.CREATED) successCount.incrementAndGet(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - indexThreads[t].start(); - } - - // Flush thread — flushes while indexing is in progress - Thread flushThread = new Thread(() -> { - try { - barrier.await(); - for (int i = 0; i < 5; i++) { - Thread.sleep(10); - flushIndex(indexName); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - flushThread.start(); - - for (Thread t : indexThreads) { - t.join(); - } - flushThread.join(); - - // Final flush to commit any remaining buffered data - flushIndex(indexName); - DataFormatAwareEngine engine = getEngine(indexName); - engine.refresh("verify"); - - int expected = successCount.get(); - long parquetRows = getRowCount(indexName, "parquet"); - long luceneRows = getRowCount(indexName, "lucene"); - assertEquals("parquet must have all successful docs", expected, parquetRows); - assertEquals("lucene must have all successful docs", expected, luceneRows); - } - - // ── Helpers for Parquet+Lucene tests ── - - private void createParquetLuceneIndex(String indexName) { - CompositeEngineHelper.createCompositeIndexWithMapping(this, indexName, "parquet", Settings.EMPTY, "lucene"); - } - - private void indexDocsTo(String indexName, int count) { - CompositeEngineHelper.indexDocs(this, indexName, count); - } - - private void flushIndex(String indexName) { - CompositeEngineHelper.flush(this, indexName); - } - - private DataFormatAwareEngine getEngine(String indexName) { - return CompositeEngineHelper.getEngine(clusterService(), internalCluster(), indexName); - } - - private long getRowCount(String indexName, String formatName) throws IOException { - return CompositeEngineHelper.getRowCount(clusterService(), internalCluster(), indexName, formatName); - } - - private void assertParquetLuceneRowCountsMatch(String indexName, long expected) throws IOException { - DataFormatAwareEngine engine = getEngine(indexName); - engine.refresh("test"); - assertEquals("parquet row count", expected, getRowCount(indexName, "parquet")); - assertEquals("lucene row count", expected, getRowCount(indexName, "lucene")); - - CompositeEngineHelper.assertPerSegmentRowCountsMatch(engine, "parquet", "lucene"); - - // Checkpoint must be consistent with seqNo - long maxSeq = engine.getSeqNoStats(-1).getMaxSeqNo(); - long processedCp = engine.getProcessedLocalCheckpoint(); - assertEquals("processed checkpoint must equal maxSeqNo", maxSeq, processedCp); - } - public void testCompositeIndexUsesClusterDefaultFormatsWhenOverridesAbsent() throws IOException { String indexName = "test-composite-cluster-default"; @@ -762,7 +265,7 @@ public void testCompositeIndexUsesClusterDefaultFormatsWhenOverridesAbsent() thr .indices() .prepareCreate(indexName) .setSettings(indexSettings) - .setMapping("field_text", "type=text", "field_keyword", "type=keyword", "field_number", "type=integer") + .setMapping("field_text", "type=text,index=false", "field_keyword", "type=keyword,index=false", "field_number", "type=integer") .get(); assertTrue("Index creation should be acknowledged", response.isAcknowledged()); @@ -827,8 +330,8 @@ public void testCompositeIndexRequestOverrideBeatsClusterDefault() throws IOExce .prepareUpdateSettings() .setTransientSettings( Settings.builder() - .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet") - .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "lucene") + .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "lucene") + .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "parquet") ) .get(); @@ -837,7 +340,7 @@ public void testCompositeIndexRequestOverrideBeatsClusterDefault() throws IOExce .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") - .put("index.composite.primary_data_format", "lucene") + .put("index.composite.primary_data_format", "parquet") .putList("index.composite.secondary_data_formats") .build(); @@ -845,7 +348,14 @@ public void testCompositeIndexRequestOverrideBeatsClusterDefault() throws IOExce .indices() .prepareCreate(indexName) .setSettings(indexSettings) - .setMapping("field_text", "type=text", "field_keyword", "type=keyword", "field_number", "type=integer") + .setMapping( + "field_text", + "type=text,index=false,store=true", + "field_keyword", + "type=keyword,index=false", + "field_number", + "type=integer" + ) .get(); assertTrue("Index creation should be acknowledged", response.isAcknowledged()); @@ -853,7 +363,7 @@ public void testCompositeIndexRequestOverrideBeatsClusterDefault() throws IOExce GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings(indexName).get(); Settings actual = settingsResponse.getIndexToSettings().get(indexName); - assertEquals("lucene", actual.get(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey())); + assertEquals("parquet", actual.get(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey())); assertTrue(actual.getAsList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey()).isEmpty()); for (int i = 0; i < 10; i++) { @@ -887,7 +397,7 @@ public void testCompositeIndexRequestOverrideBeatsClusterDefault() throws IOExce commitStats.getUserData().get(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY), Function.identity() ); - assertEquals(Set.of("lucene"), snapshot.getDataFormats()); + assertEquals(Set.of("parquet"), snapshot.getDataFormats()); ensureGreen(indexName); diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetRowGroupSettingsIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetRowGroupSettingsIT.java index 58cb123e2cc67..fe674cc2bae04 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetRowGroupSettingsIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetRowGroupSettingsIT.java @@ -130,7 +130,7 @@ private void createCompositeIndex(String indexName, Settings extraSettings) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") + .putList("index.composite.secondary_data_formats", "lucene") .put(extraSettings); client().admin() diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetSettingsValidationIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetSettingsValidationIT.java index 6a0107cec8aaa..b164d4b0864bc 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetSettingsValidationIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetSettingsValidationIT.java @@ -9,19 +9,26 @@ package org.opensearch.composite; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.composite.framework.ParquetOnlyDataFormatPlugin; import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; import org.opensearch.parquet.ParquetSettings; import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,6 +42,17 @@ public class CompositeParquetSettingsValidationIT extends AbstractCompositeEngin private static final String INDEX_NAME = "test-settings-validation"; + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + ParquetOnlyDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class + ); + } + // --- Field-level encoding validation --- public void testValidFieldEncodingAccepted() { @@ -497,7 +515,7 @@ private void createCompositeIndexWithSettings(Settings extraSettings) { .indices() .prepareCreate(INDEX_NAME) .setSettings(builder) - .setMapping("name", "type=keyword", "value", "type=integer") + .setMapping("name", "type=keyword,index=false", "value", "type=integer") .get(); } @@ -516,7 +534,7 @@ private void createCompositeIndexWithNodeSettings(Settings nodeSettings) { .indices() .prepareCreate(INDEX_NAME) .setSettings(builder) - .setMapping("name", "type=keyword", "value", "type=integer") + .setMapping("name", "type=keyword,index=false", "value", "type=integer") .get(); ensureGreen(INDEX_NAME); } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedIT.java index 4bafedbcce2c9..3e2dab4583668 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedIT.java @@ -10,51 +10,23 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.NIOFSDirectory; -import org.opensearch.action.admin.indices.flush.FlushResponse; -import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.arrow.allocator.ArrowBasePlugin; import org.opensearch.be.datafusion.DataFusionPlugin; import org.opensearch.be.lucene.LucenePlugin; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.SuppressForbidden; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexService; -import org.opensearch.index.engine.CommitStats; -import org.opensearch.index.engine.dataformat.DocumentInput; -import org.opensearch.index.engine.exec.Segment; -import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.coord.DataformatAwareCatalogSnapshot; -import org.opensearch.index.shard.IndexShard; -import org.opensearch.indices.IndicesService; import org.opensearch.parquet.ParquetDataFormatPlugin; -import org.opensearch.parquet.bridge.ParquetFileMetadata; -import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; /** * Integration test for composite refresh (flush) with sort columns configured. @@ -65,9 +37,7 @@ */ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) -public class CompositeRefreshSortedIT extends OpenSearchIntegTestCase { - - private static final String INDEX_NAME = "test-composite-refresh-sorted"; +public class CompositeRefreshSortedIT extends AbstractSortedRefreshIT { @Override protected Collection> nodePlugins() { @@ -82,51 +52,10 @@ protected Collection> nodePlugins() { ); } - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) - .build(); - } - - @Override - public void tearDown() throws Exception { - try { - client().admin().indices().prepareDelete(INDEX_NAME).get(); - } catch (Exception e) { - // index may not exist - } - super.tearDown(); - } - // ══════════════════════════════════════════════════════════════════════ // Tests // ══════════════════════════════════════════════════════════════════════ - /** - * Verifies that a single flush with sort columns produces a Parquet file - * sorted by age DESC (nulls first), name ASC (nulls last). - */ - public void testSortedRefreshProducesSortedParquet() throws Exception { - createIndex(sortedParquetOnlySettings()); - - // Index documents in deliberately unsorted order - indexDoc("charlie", 30); - indexDoc("alice", 50); - indexDoc("bob", 10); - indexDoc("dave", 50); - indexDoc("eve", 30); - - flushAndRefresh(); - - DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); - assertEquals(Set.of("parquet"), snapshot.getDataFormats()); - verifyParquetRowCount(snapshot, 5); - verifyParquetSortOrder(snapshot); - verifyParquetRowIdSequential(snapshot); - } - /** * Verifies sorted refresh with Lucene secondary: * - Parquet is sorted @@ -160,31 +89,6 @@ public void testSortedRefreshWithLuceneSecondary() throws Exception { verifyParquetAndLuceneRowsAlignedSequentially(snapshot); } - /** - * Verifies that multiple flush cycles produce independently sorted segments. - */ - public void testMultipleSortedRefreshesProduceIndependentlySortedSegments() throws Exception { - createIndex(sortedParquetOnlySettings()); - - // First batch - indexDoc("zara", 5); - indexDoc("alice", 100); - indexDoc("bob", 50); - flushAndRefresh(); - - // Second batch - indexDoc("xavier", 200); - indexDoc("yolanda", 1); - indexDoc("wendy", 75); - flushAndRefresh(); - - DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); - assertEquals("Should have 2 segments", 2, snapshot.getSegments().size()); - verifyParquetRowCount(snapshot, 6); - // Each segment should be independently sorted - verifyParquetSortOrder(snapshot); - } - /** * Verifies null handling in sorted output: age DESC with nulls first, * name ASC with nulls last. Runs against Parquet primary + Lucene secondary @@ -285,30 +189,6 @@ public void testSortedRefreshWithRandomizedData() throws Exception { verifyLuceneRowIdSequential(); } - /** - * Parquet-only refresh without any sort configuration. Confirms the - * non-sort flush path still emits sequential {@code __row_id__} values - * and the expected row count. There is no permutation produced by Parquet, - * so this exercises the {@code FlushInput.EMPTY} flow end-to-end. - */ - public void testUnsortedRefreshParquetOnly() throws Exception { - createIndex(unsortedParquetOnlySettings()); - - // Insertion order is preserved end-to-end since no sort is configured. - indexDoc("charlie", 30); - indexDoc("alice", 50); - indexDoc("bob", 10); - indexDoc("dave", 50); - indexDoc("eve", 30); - - flushAndRefresh(); - - DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); - assertEquals(Set.of("parquet"), snapshot.getDataFormats()); - verifyParquetRowCount(snapshot, 5); - verifyParquetRowIdSequential(snapshot); - } - /** * Parquet primary + Lucene secondary without sort. Without a sort * permutation, Parquet does not produce a {@code RowIdMapping} and the @@ -355,21 +235,6 @@ public void testUnsortedRefreshWithLuceneSecondary() throws Exception { // Helpers: settings // ══════════════════════════════════════════════════════════════════════ - private Settings sortedParquetOnlySettings() { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.refresh_interval", "-1") - .put("index.pluggable.dataformat.enabled", true) - .put("index.pluggable.dataformat", "composite") - .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") - .putList("index.sort.field", "age", "name") - .putList("index.sort.order", "desc", "asc") - .putList("index.sort.missing", "_first", "_last") - .build(); - } - private Settings sortedParquetWithLuceneSettings() { return Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) @@ -385,18 +250,6 @@ private Settings sortedParquetWithLuceneSettings() { .build(); } - private Settings unsortedParquetOnlySettings() { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.refresh_interval", "-1") - .put("index.pluggable.dataformat.enabled", true) - .put("index.pluggable.dataformat", "composite") - .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats") - .build(); - } - private Settings unsortedParquetWithLuceneSettings() { return Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) @@ -408,506 +261,4 @@ private Settings unsortedParquetWithLuceneSettings() { .putList("index.composite.secondary_data_formats", "lucene") .build(); } - - private Settings unsortedLuceneOnlySettings() { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.refresh_interval", "-1") - .put("index.pluggable.dataformat.enabled", true) - .put("index.pluggable.dataformat", "composite") - .put("index.composite.primary_data_format", "lucene") - .putList("index.composite.secondary_data_formats") - .build(); - } - - private Settings sortedLuceneOnlySettings() { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.refresh_interval", "-1") - .put("index.pluggable.dataformat.enabled", true) - .put("index.pluggable.dataformat", "composite") - .put("index.composite.primary_data_format", "lucene") - .putList("index.composite.secondary_data_formats") - .putList("index.sort.field", "age") - .putList("index.sort.order", "desc") - .putList("index.sort.missing", "_first") - .build(); - } - - // ══════════════════════════════════════════════════════════════════════ - // Helpers: indexing - // ══════════════════════════════════════════════════════════════════════ - - private void createIndex(Settings settings) { - client().admin() - .indices() - .prepareCreate(INDEX_NAME) - .setSettings(settings) - .setMapping("name", "type=keyword", "age", "type=integer", "tag", "type=keyword") - .get(); - ensureGreen(INDEX_NAME); - } - - private void indexDoc(String name, int age) { - IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name, "age", age).get(); - assertEquals(RestStatus.CREATED, response.status()); - } - - private void indexDoc(String name, int age, String tag) { - IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name, "age", age, "tag", tag).get(); - assertEquals(RestStatus.CREATED, response.status()); - } - - private void indexDocNullAge(String name) { - IndexResponse response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", name).get(); - assertEquals(RestStatus.CREATED, response.status()); - } - - private void flushAndRefresh() { - RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(INDEX_NAME).get(); - assertEquals(RestStatus.OK, refreshResponse.getStatus()); - FlushResponse flushResponse = client().admin().indices().prepareFlush(INDEX_NAME).setForce(true).get(); - assertEquals(RestStatus.OK, flushResponse.getStatus()); - } - - // ══════════════════════════════════════════════════════════════════════ - // Helpers: verification - // ══════════════════════════════════════════════════════════════════════ - - private void verifyParquetRowCount(DataformatAwareCatalogSnapshot snapshot, int expectedTotalDocs) throws IOException { - Path parquetDir = getParquetDir(); - long totalRows = 0; - for (Segment segment : snapshot.getSegments()) { - WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); - assertNotNull("Segment should have parquet files", wfs); - for (String file : wfs.files()) { - Path filePath = parquetDir.resolve(file); - assertTrue("Parquet file should exist: " + filePath, Files.exists(filePath)); - ParquetFileMetadata metadata = RustBridge.getFileMetadata(filePath.toString()); - totalRows += metadata.numRows(); - } - } - assertEquals("Total rows should match ingested docs", expectedTotalDocs, totalRows); - } - - @SuppressForbidden(reason = "JSON parsing for sort order verification") - private void verifyParquetSortOrder(DataformatAwareCatalogSnapshot snapshot) throws Exception { - Path parquetDir = getParquetDir(); - for (Segment segment : snapshot.getSegments()) { - WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); - for (String file : wfs.files()) { - Path filePath = parquetDir.resolve(file); - String json = RustBridge.readAsJson(filePath.toString()); - List> rows; - try ( - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - json - ) - ) { - rows = parser.list().stream().map(o -> { - @SuppressWarnings("unchecked") - Map m = (Map) o; - return m; - }).toList(); - } - if (rows.size() <= 1) continue; - - for (int i = 1; i < rows.size(); i++) { - Object prevAge = rows.get(i - 1).get("age"); - Object currAge = rows.get(i).get("age"); - - // nulls first for age - if (prevAge == null && currAge == null) continue; - if (prevAge == null) continue; - if (currAge == null) { - fail("age null should come before non-null at row " + i); - } - - int prevAgeVal = ((Number) prevAge).intValue(); - int currAgeVal = ((Number) currAge).intValue(); - - assertTrue( - "age should be DESC but found " + prevAgeVal + " before " + currAgeVal + " at row " + i + " in " + file, - prevAgeVal >= currAgeVal - ); - - // When age is equal, verify name ASC (nulls last) - if (prevAgeVal == currAgeVal) { - Object prevName = rows.get(i - 1).get("name"); - Object currName = rows.get(i).get("name"); - - if (prevName != null && currName == null) continue; - if (prevName == null && currName != null) { - fail("name nulls should be last at row " + i + " in " + file); - } - if (prevName != null && currName != null) { - assertTrue( - "name should be ASC but found '" + prevName + "' before '" + currName + "' at row " + i + " in " + file, - ((String) prevName).compareTo((String) currName) <= 0 - ); - } - } - } - } - } - } - - private void verifyLuceneDocCount(int expectedTotalDocs) throws IOException { - Path luceneDir = getLuceneDir(); - assertTrue("Lucene directory should exist", Files.exists(luceneDir)); - try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { - assertEquals("Lucene doc count should match", expectedTotalDocs, reader.numDocs()); - } - } - - /** - * Verifies Parquet sort order for a dynamic set of integer sort fields. - * Supports any combination of ASC/DESC with configurable null placement. - */ - @SuppressForbidden(reason = "JSON parsing for sort order verification") - private void verifyParquetSortOrderMultiField( - DataformatAwareCatalogSnapshot snapshot, - String[] fieldNames, - String[] sortOrders, - String[] sortMissing - ) throws Exception { - Path parquetDir = getParquetDir(); - for (Segment segment : snapshot.getSegments()) { - WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); - for (String file : wfs.files()) { - Path filePath = parquetDir.resolve(file); - String json = RustBridge.readAsJson(filePath.toString()); - List> rows; - try ( - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - json - ) - ) { - rows = parser.list().stream().map(o -> { - @SuppressWarnings("unchecked") - Map m = (Map) o; - return m; - }).toList(); - } - if (rows.size() <= 1) continue; - - for (int i = 1; i < rows.size(); i++) { - int cmp = compareRowsByMultiField(rows.get(i - 1), rows.get(i), fieldNames, sortOrders, sortMissing); - assertTrue( - "Sort order violated at row " + i + " in " + file + ": prev=" + rows.get(i - 1) + " curr=" + rows.get(i), - cmp <= 0 - ); - } - } - } - } - - /** - * Compares two rows by the multi-field sort key. Returns negative if prev comes before curr - * (correct order), zero if equal, positive if prev comes after curr (violation). - */ - private int compareRowsByMultiField( - Map prev, - Map curr, - String[] fieldNames, - String[] sortOrders, - String[] sortMissing - ) { - for (int f = 0; f < fieldNames.length; f++) { - Object prevVal = prev.get(fieldNames[f]); - Object currVal = curr.get(fieldNames[f]); - boolean isAsc = "asc".equals(sortOrders[f]); - boolean nullsFirst = "_first".equals(sortMissing[f]); - - int cmp = compareValues(prevVal, currVal, isAsc, nullsFirst); - if (cmp != 0) return cmp; - } - return 0; - } - - /** - * Compares two nullable integer values according to sort direction and null placement. - * Returns negative if in correct order, zero if equal, positive if violated. - */ - private int compareValues(Object prev, Object curr, boolean isAsc, boolean nullsFirst) { - if (prev == null && curr == null) return 0; - if (prev == null) { - // prev is null: correct if nullsFirst, violation if nullsLast - return nullsFirst ? -1 : 1; - } - if (curr == null) { - // curr is null: violation if nullsFirst, correct if nullsLast - return nullsFirst ? 1 : -1; - } - int prevInt = ((Number) prev).intValue(); - int currInt = ((Number) curr).intValue(); - int natural = Integer.compare(prevInt, currInt); - return isAsc ? natural : -natural; - } - - private void verifyLuceneRowIdSequential() throws IOException { - Path luceneDir = getLuceneDir(); - try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { - for (LeafReaderContext ctx : reader.leaves()) { - SortedNumericDocValues rowIdDV = ctx.reader().getSortedNumericDocValues(DocumentInput.ROW_ID_FIELD); - if (rowIdDV == null) continue; - - long expectedRowId = 0; - for (int doc = 0; doc < ctx.reader().maxDoc(); doc++) { - if (rowIdDV.advanceExact(doc)) { - long rowId = rowIdDV.nextValue(); - assertEquals( - DocumentInput.ROW_ID_FIELD - + " should be sequential, expected " - + expectedRowId - + " but got " - + rowId - + " at doc " - + doc, - expectedRowId, - rowId - ); - expectedRowId++; - } - } - } - } - } - - /** - * Verifies that the Lucene segment is physically sorted by the configured IndexSort - * (age DESC nulls first). Reads age from sorted numeric doc values. - */ - private void verifyLuceneSortOrder() throws IOException { - Path luceneDir = getLuceneDir(); - try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { - for (LeafReaderContext ctx : reader.leaves()) { - org.apache.lucene.index.LeafReader leaf = ctx.reader(); - int maxDoc = leaf.maxDoc(); - if (maxDoc <= 1) continue; - - SortedNumericDocValues ageDV = leaf.getSortedNumericDocValues("age"); - - Long prevAge = null; - boolean prevAgeNull = true; - for (int doc = 0; doc < maxDoc; doc++) { - Long age = null; - if (ageDV != null && ageDV.advanceExact(doc)) { - age = ageDV.nextValue(); - } - - if (doc > 0) { - // age DESC nulls first - if (prevAgeNull && age == null) { - // both null — ok - } else if (prevAgeNull) { - // prev was null, current is non-null — ok (nulls first) - } else if (age == null) { - fail("null age at doc " + doc + " came after non-null (expected nulls first)"); - } else { - assertTrue("age should be DESC at doc " + doc + ", got " + prevAge + " before " + age, prevAge >= age); - } - } - prevAge = age; - prevAgeNull = (age == null); - } - } - } - } - - @SuppressForbidden(reason = "JSON parsing for row ID verification") - private void verifyParquetRowIdSequential(DataformatAwareCatalogSnapshot snapshot) throws Exception { - Path parquetDir = getParquetDir(); - for (Segment segment : snapshot.getSegments()) { - WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); - for (String file : wfs.files()) { - Path filePath = parquetDir.resolve(file); - String json = RustBridge.readAsJson(filePath.toString()); - List> rows; - try ( - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - json - ) - ) { - rows = parser.list().stream().map(o -> { - @SuppressWarnings("unchecked") - Map m = (Map) o; - return m; - }).toList(); - } - - long expectedRowId = 0; - for (Map row : rows) { - Object rowIdObj = row.get(DocumentInput.ROW_ID_FIELD); - assertNotNull( - DocumentInput.ROW_ID_FIELD + " should be present in parquet row " + expectedRowId + " in " + file, - rowIdObj - ); - long rowId = ((Number) rowIdObj).longValue(); - assertEquals( - DocumentInput.ROW_ID_FIELD - + " should be sequential, expected " - + expectedRowId - + " but got " - + rowId - + " in " - + file, - expectedRowId, - rowId - ); - expectedRowId++; - } - } - } - } - - /** - * Reads Parquet rows and Lucene documents in physical (storage) order from the - * single-shard, single-segment index and asserts that at every position i, - * Parquet row i and Lucene doc i agree on keyword fields ({@code name} and - * {@code tag}). - *

    - * The Lucene secondary writer only persists text/keyword fields (in the inverted - * index) and {@code __row_id__} (as doc values). Numeric fields like {@code age} - * are intentionally NOT stored in the Lucene secondary — they live only in - * Parquet. Keyword fields are also stored only in the inverted index (no doc - * values), so we read them via TermsEnum/postings and reconstruct the per-doc - * value. - *

    - * The {@code tag} field is NOT part of the sort key — verifying it confirms the - * row ID rewrite correctly co-locates non-sort fields across formats too. - */ - @SuppressForbidden(reason = "JSON parsing for cross-format row alignment") - private void verifyParquetAndLuceneRowsAlignedSequentially(DataformatAwareCatalogSnapshot snapshot) throws Exception { - // Read Parquet rows in physical order - Path parquetDir = getParquetDir(); - List> parquetRows = new ArrayList<>(); - for (Segment segment : snapshot.getSegments()) { - WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); - for (String file : wfs.files()) { - String json = RustBridge.readAsJson(parquetDir.resolve(file).toString()); - try ( - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - json - ) - ) { - for (Object obj : parser.list()) { - @SuppressWarnings("unchecked") - Map row = (Map) obj; - parquetRows.add(row); - } - } - } - } - - // Reconstruct per-doc keyword field values from Lucene's inverted index - Path luceneDir = getLuceneDir(); - try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { - assertEquals("Test assumes a single Lucene leaf for sequential alignment", 1, reader.leaves().size()); - org.apache.lucene.index.LeafReader leaf = reader.leaves().get(0).reader(); - - String[] luceneNames = readKeywordValuesPerDoc(leaf, "name"); - String[] luceneTags = readKeywordValuesPerDoc(leaf, "tag"); - - assertEquals("Parquet and Lucene must have same row count", parquetRows.size(), luceneNames.length); - - for (int i = 0; i < parquetRows.size(); i++) { - Map pq = parquetRows.get(i); - String pqName = (String) pq.get("name"); - String pqTag = (String) pq.get("tag"); - assertEquals("name mismatch at position " + i, pqName, luceneNames[i]); - assertEquals("tag mismatch at position " + i, pqTag, luceneTags[i]); - } - } - } - - /** - * Asserts that the {@code name} keyword field, read in Lucene doc order from - * the single-segment index, matches the given expected sequence. Used by - * unsorted-path tests to confirm insertion order is preserved end-to-end. - */ - private void verifyLuceneNamesInOrder(String[] expectedNames) throws IOException { - Path luceneDir = getLuceneDir(); - try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) { - assertEquals("Test assumes a single Lucene leaf for sequential alignment", 1, reader.leaves().size()); - org.apache.lucene.index.LeafReader leaf = reader.leaves().get(0).reader(); - String[] actual = readKeywordValuesPerDoc(leaf, "name"); - assertEquals("Lucene doc count should match expected sequence length", expectedNames.length, actual.length); - for (int i = 0; i < expectedNames.length; i++) { - assertEquals("name at lucene docId " + i, expectedNames[i], actual[i]); - } - } - } - - /** - * Reconstructs the per-doc keyword value for the given field by iterating - * the inverted index. Each doc is expected to have at most one term for the - * field. Returns an array indexed by Lucene doc ID. - */ - private String[] readKeywordValuesPerDoc(org.apache.lucene.index.LeafReader leaf, String fieldName) throws IOException { - String[] values = new String[leaf.maxDoc()]; - org.apache.lucene.index.Terms terms = leaf.terms(fieldName); - assertNotNull("Lucene index should have field '" + fieldName + "' in inverted index", terms); - org.apache.lucene.index.TermsEnum it = terms.iterator(); - org.apache.lucene.util.BytesRef term; - while ((term = it.next()) != null) { - String value = term.utf8ToString(); - org.apache.lucene.index.PostingsEnum postings = it.postings(null); - int doc; - while ((doc = postings.nextDoc()) != org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS) { - values[doc] = value; - } - } - return values; - } - - // ══════════════════════════════════════════════════════════════════════ - // Helpers: accessors - // ══════════════════════════════════════════════════════════════════════ - - private DataformatAwareCatalogSnapshot getCatalogSnapshot() throws IOException { - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, getNodeName()); - IndexService indexService = indicesService.indexServiceSafe(resolveIndex(INDEX_NAME)); - IndexShard shard = indexService.getShard(0); - CommitStats commitStats = shard.commitStats(); - assertNotNull(commitStats); - assertNotNull(commitStats.getUserData()); - assertTrue(commitStats.getUserData().containsKey(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY)); - return DataformatAwareCatalogSnapshot.deserializeFromString( - commitStats.getUserData().get(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY), - Function.identity() - ); - } - - private Path getParquetDir() { - IndexShard shard = getPrimaryShard(); - return shard.shardPath().getDataPath().resolve("parquet"); - } - - private Path getLuceneDir() { - IndexShard shard = getPrimaryShard(); - return shard.shardPath().resolveIndex(); - } - - private IndexShard getPrimaryShard() { - String nodeName = getNodeName(); - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName); - IndexService indexService = indicesService.indexServiceSafe(resolveIndex(INDEX_NAME)); - return indexService.getShard(0); - } - - private String getNodeName() { - String nodeId = getClusterState().routingTable().index(INDEX_NAME).shard(0).primaryShard().currentNodeId(); - return getClusterState().nodes().get(nodeId).getName(); - } } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedParquetOnlyIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedParquetOnlyIT.java new file mode 100644 index 0000000000000..6ffea2378410e --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeRefreshSortedParquetOnlyIT.java @@ -0,0 +1,142 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.composite.framework.ParquetOnlyDataFormatPlugin; +import org.opensearch.index.engine.exec.coord.DataformatAwareCatalogSnapshot; +import org.opensearch.plugins.Plugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; + +public class CompositeRefreshSortedParquetOnlyIT extends AbstractSortedRefreshIT { + + @Override + protected Collection> nodePlugins() { + // ParquetDataFormatPlugin sources its allocator from the unified native-allocator + // framework's ingest pool, so the framework plugin must be installed. + return Arrays.asList( + ArrowBasePlugin.class, + ParquetOnlyDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class + ); + } + + /** + * Verifies that a single flush with sort columns produces a Parquet file + * sorted by age DESC (nulls first), name ASC (nulls last). + */ + public void testSortedRefreshProducesSortedParquet() throws Exception { + createIndex(sortedParquetOnlySettings()); + + // Index documents in deliberately unsorted order + indexDoc("charlie", 30); + indexDoc("alice", 50); + indexDoc("bob", 10); + indexDoc("dave", 50); + indexDoc("eve", 30); + + flushAndRefresh(); + + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertEquals(Set.of("parquet"), snapshot.getDataFormats()); + verifyParquetRowCount(snapshot, 5); + verifyParquetSortOrder(snapshot); + verifyParquetRowIdSequential(snapshot); + } + + /** + * Verifies that multiple flush cycles produce independently sorted segments. + */ + public void testMultipleSortedRefreshesProduceIndependentlySortedSegments() throws Exception { + createIndex(sortedParquetOnlySettings()); + + // First batch + indexDoc("zara", 5); + indexDoc("alice", 100); + indexDoc("bob", 50); + flushAndRefresh(); + + // Second batch + indexDoc("xavier", 200); + indexDoc("yolanda", 1); + indexDoc("wendy", 75); + flushAndRefresh(); + + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertEquals("Should have 2 segments", 2, snapshot.getSegments().size()); + verifyParquetRowCount(snapshot, 6); + // Each segment should be independently sorted + verifyParquetSortOrder(snapshot); + } + + /** + * Parquet-only refresh without any sort configuration. Confirms the + * non-sort flush path still emits sequential {@code __row_id__} values + * and the expected row count. There is no permutation produced by Parquet, + * so this exercises the {@code FlushInput.EMPTY} flow end-to-end. + */ + public void testUnsortedRefreshParquetOnly() throws Exception { + createIndex(unsortedParquetOnlySettings()); + + // Insertion order is preserved end-to-end since no sort is configured. + indexDoc("charlie", 30); + indexDoc("alice", 50); + indexDoc("bob", 10); + indexDoc("dave", 50); + indexDoc("eve", 30); + + flushAndRefresh(); + + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertEquals(Set.of("parquet"), snapshot.getDataFormats()); + verifyParquetRowCount(snapshot, 5); + verifyParquetRowIdSequential(snapshot); + } + + private Settings sortedParquetOnlySettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.refresh_interval", "-1") + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .putList("index.sort.field", "age", "name") + .putList("index.sort.order", "desc", "asc") + .putList("index.sort.missing", "_first", "_last") + .build(); + } + + private Settings unsortedParquetOnlySettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.refresh_interval", "-1") + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + } + + @Override + protected String[] getMapping() { + return new String[] { "name", "type=keyword,index=false", "age", "type=integer", "tag", "type=keyword,index=false" }; + } +} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryIT.java index 7ecb618295ea0..f7f1190d512be 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryIT.java @@ -136,7 +136,7 @@ public void testMultiShardRecoveryNoRedState() throws Exception { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats", List.of()) + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); client().admin().indices().prepareCreate(INDEX_NAME).setSettings(multiShardSettings).get(); diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryWithLuceneIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryWithLuceneIT.java deleted file mode 100644 index 9f736cac1da7e..0000000000000 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareMultiIndexRecoveryWithLuceneIT.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.opensearch.common.settings.Settings; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Set; - -/** - * Runs all multi-index recovery tests from {@link DataFormatAwareMultiIndexRecoveryIT} with - * Lucene as a secondary data format alongside Parquet. - */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class DataFormatAwareMultiIndexRecoveryWithLuceneIT extends DataFormatAwareMultiIndexRecoveryIT { - - @Override - protected Settings dfaIndexSettings(int replicaCount) { - return Settings.builder() - .put(super.dfaIndexSettings(replicaCount)) - .putList("index.composite.secondary_data_formats", java.util.List.of("lucene")) - .build(); - } - - @Override - protected boolean hasLuceneSecondary() { - return true; - } - - @Override - protected Set expectedFormats() { - return Set.of("parquet", "lucene"); - } -} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwarePeerRecoveryWithLuceneIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwarePeerRecoveryWithLuceneIT.java deleted file mode 100644 index 2ff1f195c96e8..0000000000000 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwarePeerRecoveryWithLuceneIT.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.opensearch.common.settings.Settings; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Set; - -/** - * Runs all peer recovery tests from {@link DataFormatAwarePeerRecoveryIT} with Lucene as a - * secondary data format alongside Parquet. - */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class DataFormatAwarePeerRecoveryWithLuceneIT extends DataFormatAwarePeerRecoveryIT { - - @Override - protected Settings dfaIndexSettings(int replicaCount) { - return Settings.builder() - .put(super.dfaIndexSettings(replicaCount)) - .putList("index.composite.secondary_data_formats", java.util.List.of("lucene")) - .build(); - } - - @Override - protected boolean hasLuceneSecondary() { - return true; - } - - @Override - protected Set expectedFormats() { - return Set.of("parquet", "lucene"); - } -} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReadonlyEngineBaseIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReadonlyEngineBaseIT.java index 4fcd88084dc04..770dded8b3089 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReadonlyEngineBaseIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReadonlyEngineBaseIT.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.framework.ParquetOnlyDataFormatPlugin; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.IndexModule; @@ -24,7 +25,6 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.Node; -import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase; @@ -59,7 +59,7 @@ protected Collection> nodePlugins() { super.nodePlugins().stream(), Stream.of( ArrowBasePlugin.class, - ParquetDataFormatPlugin.class, + ParquetOnlyDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class, DataFusionPlugin.class @@ -96,7 +96,7 @@ protected Settings dfaIndexSettings(int replicaCount) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats", List.of()) + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryIT.java index bd7e43a159072..4f4407b1a1837 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryIT.java @@ -90,7 +90,7 @@ protected Settings dfaIndexSettings(int replicaCount) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats", List.of()) + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); } @@ -101,7 +101,7 @@ protected Set expectedFormats() { /** Whether lucene is configured as a secondary data format (produces searchable segment files). */ protected boolean hasLuceneSecondary() { - return false; + return true; } /** diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryWithLuceneIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryWithLuceneIT.java deleted file mode 100644 index f95cd5613ecfd..0000000000000 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRemoteStoreRecoveryWithLuceneIT.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.opensearch.common.settings.Settings; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Set; - -/** - * Runs all recovery tests from {@link DataFormatAwareRemoteStoreRecoveryIT} with Lucene as - * a secondary data format. Validates that Lucene segment files are correctly persisted, - * restored, and survive node restart / shard relocation alongside Parquet files. - */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -@org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks(bugUrl = "parquet native dir cleanup") -public class DataFormatAwareRemoteStoreRecoveryWithLuceneIT extends DataFormatAwareRemoteStoreRecoveryIT { - - @Override - protected Settings dfaIndexSettings(int replicaCount) { - return Settings.builder() - .put(super.dfaIndexSettings(replicaCount)) - .putList("index.composite.secondary_data_formats", java.util.List.of("lucene")) - .build(); - } - - @Override - protected boolean hasLuceneSecondary() { - return true; - } - - @Override - protected Set expectedFormats() { - return Set.of("parquet", "lucene"); - } -} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationBaseIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationBaseIT.java index d9989c5e399fc..cc0e92068467d 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationBaseIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationBaseIT.java @@ -74,7 +74,7 @@ protected Settings dfaIndexSettings(int replicaCount) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats", List.of()) + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); } @@ -92,7 +92,7 @@ protected Set expectedFormats() { /** Whether lucene is configured as a secondary data format (produces searchable segment files). */ protected boolean hasLuceneSecondary() { - return false; + return true; } /** diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationIT.java index 33a0f54a1a757..f22f6d3536e4f 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationIT.java @@ -81,7 +81,7 @@ protected Settings dfaIndexSettings(int replicaCount) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats", List.of()) + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); } @@ -92,7 +92,7 @@ protected Set expectedFormats() { /** Whether lucene is configured as a secondary data format (produces searchable segment files). */ protected boolean hasLuceneSecondary() { - return false; + return true; } /** diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationPromotionWithLuceneIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationPromotionWithLuceneIT.java deleted file mode 100644 index 814e3e1a72c57..0000000000000 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationPromotionWithLuceneIT.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; -import org.opensearch.common.settings.Settings; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Set; - -/** - * Runs all promotion tests from {@link DataFormatAwareReplicationPromotionIT} with Lucene as - * a secondary data format alongside Parquet. Validates that real Lucene segment files survive - * the replication and promotion paths. - */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -@AwaitsFix(bugUrl = "flaky test - enable after fix") -public class DataFormatAwareReplicationPromotionWithLuceneIT extends DataFormatAwareReplicationPromotionIT { - - @Override - protected Settings dfaIndexSettings(int replicaCount) { - // Override to add lucene as a secondary data format. - return Settings.builder() - .put(super.dfaIndexSettings(replicaCount)) - .putList("index.composite.secondary_data_formats", java.util.List.of("lucene")) - .build(); - } - - @Override - protected boolean hasLuceneSecondary() { - return true; - } - - @Override - protected Set expectedFormats() { - return Set.of("parquet", "lucene"); - } - - @Override - @AwaitsFix(bugUrl = "Flaky test - fix before enabling") - public void testUncleanFailoverWithContinuousIndexing() throws Exception { - super.testUncleanFailoverWithContinuousIndexing(); - } -} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationWithLuceneIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationWithLuceneIT.java deleted file mode 100644 index 7a48424fa81b0..0000000000000 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareReplicationWithLuceneIT.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.opensearch.common.settings.Settings; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Set; - -/** - * Runs all replication tests from {@link DataFormatAwareReplicationIT} with Lucene as a - * secondary data format alongside Parquet. - */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class DataFormatAwareReplicationWithLuceneIT extends DataFormatAwareReplicationIT { - - @Override - protected Settings dfaIndexSettings(int replicaCount) { - return Settings.builder() - .put(super.dfaIndexSettings(replicaCount)) - .putList("index.composite.secondary_data_formats", java.util.List.of("lucene")) - .build(); - } - - @Override - protected boolean hasLuceneSecondary() { - return true; - } - - @Override - protected Set expectedFormats() { - return Set.of("parquet", "lucene"); - } -} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2IT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2IT.java index 65eabf825527d..24d7e97a05238 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2IT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2IT.java @@ -161,14 +161,14 @@ protected Settings.Builder getRepositorySettings(String sourceRepository, boolea * this to add lucene as a secondary format. */ protected List getSecondaryDataFormats() { - return List.of(); + return List.of("lucene"); } /** * Whether lucene is configured as a secondary format. Drives format-aware on-disk assertions. */ protected boolean hasLuceneSecondary() { - return false; + return true; } protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2WithLuceneIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2WithLuceneIT.java deleted file mode 100644 index 07647b3355323..0000000000000 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareRestoreShallowSnapshotV2WithLuceneIT.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.opensearch.common.settings.Settings; - -import java.util.List; - -/** - * Variant of {@link DataFormatAwareRestoreShallowSnapshotV2IT} that uses parquet as the primary - * format with lucene as a secondary format (so the Lucene index/ directory - * contains real segment data files in addition to {@code segments_N}). - * - *

    Inherits all V2 snapshot tests from the base class. The only difference is the index - * settings: {@code index.composite.secondary_data_formats} is {@code ["lucene"]} instead of - * the default empty list. This exercises the parquet+lucene-secondary code paths during - * V2 snapshot creation, restore, and validation. - * - *

    Mirrors the {@code DataFormatAwareReplicationWithLuceneIT} pattern: same test surface, - * different format combination. - * - * @opensearch.experimental - */ -public class DataFormatAwareRestoreShallowSnapshotV2WithLuceneIT extends DataFormatAwareRestoreShallowSnapshotV2IT { - - public DataFormatAwareRestoreShallowSnapshotV2WithLuceneIT(Settings nodeSettings) { - super(nodeSettings); - } - - /** Lucene is configured as a secondary format → both formats produce searchable segment files. */ - @Override - protected List getSecondaryDataFormats() { - return List.of("lucene"); - } - - /** Drives format-aware on-disk assertions: index/ must contain segment data files beyond segments_N. */ - @Override - protected boolean hasLuceneSecondary() { - return true; - } -} diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareUploadIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareUploadIT.java index 44aec58296a43..1a6fb281d7104 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareUploadIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/DataFormatAwareUploadIT.java @@ -71,7 +71,7 @@ protected Settings dfaIndexSettings(int replicaCount) { .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") - .putList("index.composite.secondary_data_formats", List.of()) + .putList("index.composite.secondary_data_formats", List.of("lucene")) .build(); } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/FailableParquetDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/FailableParquetDataFormatPlugin.java index e0c2479c9f092..240cd9d9828e7 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/FailableParquetDataFormatPlugin.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/FailableParquetDataFormatPlugin.java @@ -8,6 +8,7 @@ package org.opensearch.composite; +import org.opensearch.composite.framework.ParquetOnlyDataFormatPlugin; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.FileInfos; import org.opensearch.index.engine.dataformat.FlushInput; @@ -24,7 +25,6 @@ import org.opensearch.index.engine.exec.EngineReaderManager; import org.opensearch.index.engine.exec.commit.IndexStoreProvider; import org.opensearch.index.store.FormatChecksumStrategy; -import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.writer.ParquetDocumentInput; @@ -53,7 +53,7 @@ * FailableParquetDataFormatPlugin.clearFailure(); * } */ -public class FailableParquetDataFormatPlugin extends ParquetDataFormatPlugin { +public class FailableParquetDataFormatPlugin extends ParquetOnlyDataFormatPlugin { private static final AtomicBoolean suppressMappingUpdates = new AtomicBoolean(false); diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/framework/ParquetOnlyDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/framework/ParquetOnlyDataFormatPlugin.java new file mode 100644 index 0000000000000..033bf0b7e948f --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/framework/ParquetOnlyDataFormatPlugin.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite.framework; + +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.FieldNamesFieldMapper; +import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.IgnoredFieldMapper; +import org.opensearch.index.mapper.IndexFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; +import org.opensearch.index.mapper.RoutingFieldMapper; +import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; +import org.opensearch.index.mapper.VersionFieldMapper; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.engine.ParquetDataFormat; + +import java.util.HashSet; +import java.util.Set; + +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.POINT_RANGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.STORED_FIELDS; + +/** + * Parquet Data format when there is no secondary lucene. + * While parquet cannot support full text for meta fields, the support is added here to ensure we can run tests with just parquet format. + */ +public class ParquetOnlyDataFormatPlugin extends ParquetDataFormatPlugin { + + @Override + public DataFormat getDataFormat() { + return new ParquetDataFormat() { + @Override + public Set supportedFields() { + Set supported = new HashSet<>(super.supportedFields()); + Set metaSupported = Set.of( + new FieldTypeCapabilities(DocCountFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(RoutingFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IgnoredFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IdFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SeqNoFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, POINT_RANGE)), + new FieldTypeCapabilities(SeqNoFieldMapper.PRIMARY_TERM_NAME, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(VersionFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(IndexFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(NestedPathFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SourceFieldMapper.NAME, Set.of(STORED_FIELDS)), + new FieldTypeCapabilities(FieldNamesFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)) + ); + supported.removeIf(f -> containsMetaField(metaSupported, f.fieldType())); + supported.addAll(metaSupported); + return supported; + } + }; + } + + private boolean containsMetaField(Set fieldTypeCapabilities, String fieldType) { + for (FieldTypeCapabilities capability : fieldTypeCapabilities) { + if (capability.fieldType().equals(fieldType)) { + return true; + } + } + return false; + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java index f1757189361ad..2ca4b11ed07cc 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java @@ -28,9 +28,12 @@ import org.opensearch.index.engine.dataformat.DataFormatDescriptor; import org.opensearch.index.engine.dataformat.DataFormatPlugin; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.shard.IndexSettingProvider; import org.opensearch.indices.IndexCreationException; import org.opensearch.indices.IndicesService; @@ -45,10 +48,16 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Sandbox plugin that provides a {@link CompositeIndexingExecutionEngine} for @@ -315,6 +324,92 @@ public Map> getFormatDescriptors( return Map.copyOf(descriptors); } + private List getConfiguredFormats(IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + Settings settings = indexSettings.getSettings(); + String primaryFormatName = PRIMARY_DATA_FORMAT.get(settings); + List secondaryFormatNames = SECONDARY_DATA_FORMATS.get(settings); + + List configured = new ArrayList<>(); + if (primaryFormatName != null && primaryFormatName.isEmpty() == false) { + dataFormatRegistry.getRegisteredFormats() + .stream() + .filter(f -> f.name().equals(primaryFormatName)) + .findFirst() + .ifPresent(configured::add); + } + secondaryFormatNames.stream() + .filter(name -> name != null && name.isEmpty() == false) + .map(name -> dataFormatRegistry.getRegisteredFormats().stream().filter(f -> f.name().equals(name)).findFirst().orElse(null)) + .filter(Objects::nonNull) + .sorted(Comparator.comparingLong(DataFormat::priority)) + .forEach(configured::add); + return List.copyOf(configured); + } + + /** + * Assigns capabilities by delegating to primary format first, then secondaries in order. + * Each sub-format plugin claims the capabilities it supports; unclaimed capabilities are + * passed to the next format. + */ + @Override + public void assignCapabilities(MappedFieldType fieldType, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + Set requested = fieldType.requestedCapabilities(); + if (requested.isEmpty()) { + fieldType.setCapabilityMap(Map.of()); + return; + } + + List formats = getConfiguredFormats(indexSettings, dataFormatRegistry); + if (formats.isEmpty()) { + fieldType.setCapabilityMap(Map.of()); + return; + } + + String typeName = fieldType.typeName(); + Set remaining = new HashSet<>(requested); + Map> assigned = new HashMap<>(); + + for (DataFormat format : formats) { + if (remaining.isEmpty()) { + break; + } + Set claimed = format.supportedFields() + .stream() + .filter(ftc -> ftc.fieldType().equals(typeName)) + .findFirst() + .map(ftc -> { + Set intersection = EnumSet.noneOf(FieldTypeCapabilities.Capability.class); + for (FieldTypeCapabilities.Capability cap : remaining) { + if (ftc.capabilities().contains(cap)) { + intersection.add(cap); + } + } + return intersection; + }) + .orElse(Set.of()); + if (claimed.isEmpty() == false) { + assigned.put(format, Set.copyOf(claimed)); + remaining.removeAll(claimed); + } + } + + if (remaining.isEmpty() == false) { + throw new MapperParsingException( + "Field [" + + fieldType.name() + + "] of type [" + + typeName + + "] requires capabilities " + + requested + + " but configured data formats cannot collectively cover: " + + remaining + + ". Configured formats: " + + formats.stream().map(DataFormat::name).collect(Collectors.toList()) + ); + } + fieldType.setCapabilityMap(Map.copyOf(assigned)); + } + /** * Returns the store strategies from every participating sub-format plugin * (primary + secondary), keyed by format name. Mirrors {@link #getFormatDescriptors}: diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeAssignCapabilitiesTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeAssignCapabilitiesTests.java new file mode 100644 index 0000000000000..8c4a42e2ae454 --- /dev/null +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeAssignCapabilitiesTests.java @@ -0,0 +1,352 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link CompositeDataFormatPlugin#assignCapabilities(MappedFieldType, IndexSettings, DataFormatRegistry)}. + */ +public class CompositeAssignCapabilitiesTests extends OpenSearchTestCase { + + public void testPrimaryCoversAllCapabilities() { + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.COLUMNAR_STORAGE, Capability.FULL_TEXT_SEARCH))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet").build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + Map> map = field.getCapabilityMap(); + assertEquals(1, map.size()); + assertEquals(Set.of(Capability.COLUMNAR_STORAGE, Capability.FULL_TEXT_SEARCH), map.get(parquet)); + } + + public void testPrimaryPartialSecondaryCompletes() { + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.COLUMNAR_STORAGE))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.FULL_TEXT_SEARCH))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene") + .build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + Map> map = field.getCapabilityMap(); + assertEquals(2, map.size()); + assertEquals(Set.of(Capability.COLUMNAR_STORAGE), map.get(parquet)); + assertEquals(Set.of(Capability.FULL_TEXT_SEARCH), map.get(lucene)); + } + + public void testNeitherFormatCoversAll_Throws() { + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.BLOOM_FILTER))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.STORED_FIELDS))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene") + .build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> plugin.assignCapabilities(field, indexSettings, registry) + ); + assertTrue(ex.getMessage().contains("FULL_TEXT_SEARCH")); + assertTrue(ex.getMessage().contains("COLUMNAR_STORAGE")); + } + + public void testNoFormatsConfigured_EmptyMap() { + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of()); + + // Use a primary format name that doesn't match anything in the registry + IndexSettings indexSettings = buildIndexSettings( + Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "nonexistent").build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + assertTrue(field.getCapabilityMap().isEmpty()); + } + + public void testFieldNotSupportedByAnyFormat_Throws() { + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.POINT_RANGE, Capability.COLUMNAR_STORAGE))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.POINT_RANGE, Capability.COLUMNAR_STORAGE))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene") + .build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> plugin.assignCapabilities(field, indexSettings, registry) + ); + assertTrue(ex.getMessage().contains("keyword")); + } + + public void testAllCapabilitiesSatisfiedBySecondaryOnly() { + // Primary doesn't support keyword at all + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.POINT_RANGE))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.FULL_TEXT_SEARCH, Capability.COLUMNAR_STORAGE))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene") + .build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + Map> map = field.getCapabilityMap(); + assertEquals(1, map.size()); + assertEquals(Set.of(Capability.FULL_TEXT_SEARCH, Capability.COLUMNAR_STORAGE), map.get(lucene)); + } + + public void testPriorityOrderPrimaryClaimsFirst() { + // Both support COLUMNAR_STORAGE for keyword, but primary is first in configured order + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.COLUMNAR_STORAGE))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.COLUMNAR_STORAGE, Capability.FULL_TEXT_SEARCH))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene") + .build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + Map> map = field.getCapabilityMap(); + assertEquals(2, map.size()); + assertEquals(Set.of(Capability.COLUMNAR_STORAGE), map.get(parquet)); + assertEquals(Set.of(Capability.FULL_TEXT_SEARCH), map.get(lucene)); + } + + public void testMultipleSecondariesSplitCapabilities() { + // Use "integer" type which requests POINT_RANGE + COLUMNAR_STORAGE + STORED_FIELDS when all three are enabled + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.COLUMNAR_STORAGE))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.POINT_RANGE))) + ); + DataFormat arrow = CompositeTestHelper.stubFormat( + "arrow", + 3, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.STORED_FIELDS))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene, arrow)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene", "arrow") + .build() + ); + + // NumberFieldType with isSearchable=true, isStored=true, hasDocValues=true → POINT_RANGE + STORED_FIELDS + COLUMNAR_STORAGE + MappedFieldType field = new NumberFieldMapper.NumberFieldType( + "f", + NumberFieldMapper.NumberType.INTEGER, + true, + true, + true, + false, + true, + null, + Map.of() + ); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + Map> map = field.getCapabilityMap(); + assertEquals(3, map.size()); + assertEquals(Set.of(Capability.COLUMNAR_STORAGE), map.get(parquet)); + assertEquals(Set.of(Capability.POINT_RANGE), map.get(lucene)); + assertEquals(Set.of(Capability.STORED_FIELDS), map.get(arrow)); + } + + public void testFieldWithNoRequestedCapabilities_EmptyMap() { + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("integer", Set.of(Capability.POINT_RANGE, Capability.COLUMNAR_STORAGE))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet").build() + ); + + // No-caps field: isSearchable=false, isStored=false, hasDocValues=false + MappedFieldType field = new NumberFieldMapper.NumberFieldType( + "x", + NumberFieldMapper.NumberType.INTEGER, + false, + false, + false, + false, + true, + null, + Map.of() + ); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + plugin.assignCapabilities(field, indexSettings, registry); + + assertTrue(field.getCapabilityMap().isEmpty()); + } + + public void testPartialAcrossFormatsButStillIncomplete_Throws() { + DataFormat parquet = CompositeTestHelper.stubFormat( + "parquet", + 1, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.COLUMNAR_STORAGE))) + ); + DataFormat lucene = CompositeTestHelper.stubFormat( + "lucene", + 2, + Set.of(new FieldTypeCapabilities("keyword", Set.of(Capability.STORED_FIELDS))) + ); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + when(registry.getRegisteredFormats()).thenReturn(Set.of(parquet, lucene)); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet") + .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "lucene") + .build() + ); + + MappedFieldType field = new KeywordFieldMapper.KeywordFieldType("name"); + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> plugin.assignCapabilities(field, indexSettings, registry) + ); + assertTrue(ex.getMessage().contains("FULL_TEXT_SEARCH")); + } + + private static IndexSettings buildIndexSettings(Settings extra) { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(extra) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); + return new IndexSettings(indexMetadata, Settings.EMPTY); + } +} diff --git a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java index 8a65d5fe52734..3f826a441f98e 100644 --- a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java +++ b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java @@ -14,11 +14,13 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.fields.ArrowFieldRegistry; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.memory.ArrowBufferPool; @@ -45,6 +47,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; /** @@ -77,6 +80,7 @@ public class VSRRotationBenchmark { @Param({ "20" }) private int fieldCount; + private static final DataFormat PARQUET_FORMAT = new ParquetDataFormat(); private ThreadPool threadPool; private ArrowBufferPool bufferPool; private org.opensearch.arrow.allocator.ArrowNativeAllocator nativeAllocator; @@ -117,6 +121,11 @@ public void setupTrial() { ft = new KeywordFieldMapper.KeywordFieldType("keyword_field_" + i); break; } + PARQUET_FORMAT.supportedFields() + .stream() + .filter(ftc -> ftc.fieldType().equals(ft.typeName())) + .findFirst() + .ifPresent(ftc -> ft.setCapabilityMap(Map.of(PARQUET_FORMAT, ftc.capabilities()))); fieldTypes.add(ft); ParquetField pf = ArrowFieldRegistry.getParquetField(ft.typeName()); arrowFields.add(new Field(ft.name(), pf.getFieldType(), null)); diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetDataFormat.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetDataFormat.java index 73cc70bde5ca9..0bfe394096085 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetDataFormat.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetDataFormat.java @@ -13,7 +13,6 @@ import org.opensearch.parquet.fields.ArrowFieldRegistry; import java.util.Set; -import java.util.stream.Collectors; /** * Data format descriptor for the Parquet format. @@ -42,16 +41,6 @@ public long priority() { @Override public Set supportedFields() { - // TODO - Override FieldRegistry to return capability for each field - return ArrowFieldRegistry.getRegisteredFields() - .keySet() - .stream() - .map( - type -> new FieldTypeCapabilities( - type, - Set.of(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, FieldTypeCapabilities.Capability.BLOOM_FILTER) - ) - ) - .collect(Collectors.toUnmodifiableSet()); + return ArrowFieldRegistry.getSupportedFieldCapabilities(); } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowFieldRegistry.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowFieldRegistry.java index 47a7df28103b7..7333945634a43 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowFieldRegistry.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowFieldRegistry.java @@ -8,13 +8,16 @@ package org.opensearch.parquet.fields; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.parquet.fields.plugins.CoreDataFieldPlugin; import org.opensearch.parquet.fields.plugins.MetadataFieldPlugin; import org.opensearch.parquet.fields.plugins.ParquetFieldPlugin; import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; /** * Registry mapping OpenSearch field types to their corresponding Parquet field implementations. @@ -40,7 +43,7 @@ private static void registerPlugin(ParquetFieldPlugin plugin, String pluginName) if (fields == null || fields.isEmpty()) { return; } - for (Map.Entry entry : fields.entrySet()) { + for (Map.Entry entry : fields.entrySet()) { String fieldType = entry.getKey(); ParquetField parquetField = entry.getValue(); if (fieldType == null || fieldType.trim().isEmpty()) { @@ -67,6 +70,19 @@ public static ParquetField getParquetField(String fieldType) { return FIELD_REGISTRY.get(fieldType); } + /** + * Returns the supported field type capabilities for all registered fields, + * querying each {@link ParquetField} for its supported capabilities. + * + * @return unmodifiable set of {@link FieldTypeCapabilities} for all registered field types + */ + public static Set getSupportedFieldCapabilities() { + return FIELD_REGISTRY.entrySet() + .stream() + .map(e -> new FieldTypeCapabilities(e.getKey(), e.getValue().supportedCapabilities())) + .collect(Collectors.toUnmodifiableSet()); + } + /** * Returns an unmodifiable view of all registered fields. * @return map of field type names to ParquetField implementations diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java index b8283f30f809d..f9718af73f820 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java @@ -13,8 +13,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.index.engine.dataformat.DocumentInput; +import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.IndexFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NestedPathFieldMapper; @@ -41,18 +43,20 @@ private ArrowSchemaBuilder() {} * TODO - Get the mapping version while creating the schema */ public static Schema getSchema(MapperService mapperService) { - Objects.requireNonNull(mapperService, "MapperService cannot be null"); List fields = new ArrayList<>(); - if (mapperService.documentMapper() != null) { - for (Mapper mapper : mapperService.documentMapperWithAutoCreate().getDocumentMapper().mappers()) { + DocumentMapper documentMapper = mapperService.documentMapperWithAutoCreate().getDocumentMapper(); + if (documentMapper != null) { + for (Mapper mapper : documentMapper.mappers()) { if (isUnsupportedMetadataField(mapper)) { logger.debug("Skipping unsupported metadata field: [{}] of type [{}]", mapper.name(), mapper.typeName()); continue; } + ParquetField parquetField = ArrowFieldRegistry.getParquetField(mapper.typeName()); if (parquetField != null) { fields.add(new Field(mapper.name(), parquetField.getFieldType(), null)); + handleNormalizedField(mapper, documentMapper, fields, parquetField); } else { logger.debug("No ParquetField registered for field: [{}] of type [{}]", mapper.name(), mapper.typeName()); } @@ -65,6 +69,15 @@ public static Schema getSchema(MapperService mapperService) { return new Schema(fields); } + private static void handleNormalizedField(Mapper mapper, DocumentMapper documentMapper, List fields, ParquetField parquetField) { + if (mapper instanceof KeywordFieldMapper keywordFieldMapper) { + if (!documentMapper.mappers().isMultiField(mapper.name()) && keywordFieldMapper.getRawValueFieldType() != null) { + KeywordFieldMapper.KeywordFieldType rawValueField = keywordFieldMapper.getRawValueFieldType(); + fields.add(new Field(rawValueField.name(), parquetField.getFieldType(), null)); + } + } + } + private static boolean isUnsupportedMetadataField(Mapper mapper) { return mapper instanceof SourceFieldMapper || mapper instanceof FieldNamesFieldMapper diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java index d1cdd67165240..099c303c8282f 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java @@ -10,9 +10,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.vsr.ManagedVSR; +import java.util.Set; + /** * Abstract base class for Parquet field implementations that handle conversion * between OpenSearch field types and Apache Arrow vectors. @@ -42,6 +45,16 @@ public final void createField(MappedFieldType fieldType, ManagedVSR managedVSR, addToGroup(fieldType, managedVSR, parseValue); } + /** + * Returns the set of capabilities supported by this field type. + * Subclasses may override to declare different capabilities. + * + * @return set of supported {@link FieldTypeCapabilities.Capability} + */ + public Set supportedCapabilities() { + return Set.of(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, FieldTypeCapabilities.Capability.BLOOM_FILTER); + } + /** Returns the Arrow type for this field. */ public abstract ArrowType getArrowType(); diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BinaryParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BinaryParquetField.java index 69b5e3aacdbbc..e2f5fe0129b93 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BinaryParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BinaryParquetField.java @@ -11,10 +11,13 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; +import java.util.Set; + /** * Parquet field for binary data using {@link VarBinaryVector}. */ @@ -37,4 +40,9 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, FieldTypeCapabilities.Capability.STORED_FIELDS); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BooleanParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BooleanParquetField.java index 877bb82ad9a26..75860ccb262d7 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BooleanParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/BooleanParquetField.java @@ -11,10 +11,13 @@ import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; +import java.util.Set; + /** * Parquet field for boolean values using {@link BitVector}. */ @@ -33,6 +36,15 @@ public ArrowType getArrowType() { return new ArrowType.Bool(); } + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH // This can be supported directly via stats + ); + } + @Override public FieldType getFieldType() { return FieldType.nullable(getArrowType()); diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateNanosParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateNanosParquetField.java index e378b2ed57ff5..5ecdf56f7e5ea 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateNanosParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateNanosParquetField.java @@ -12,10 +12,13 @@ import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; +import java.util.Set; + /** * Parquet field for date_nanos values stored as nanosecond timestamps using {@link TimeStampNanoVector}. */ @@ -38,4 +41,13 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.POINT_RANGE + ); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateParquetField.java index fd1b5288320bf..1586bad6bfa25 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/date/DateParquetField.java @@ -12,10 +12,13 @@ import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; +import java.util.Set; + /** * Parquet field for date values stored as millisecond timestamps using {@link TimeStampMilliVector}. */ @@ -38,4 +41,13 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.POINT_RANGE + ); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ByteParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ByteParquetField.java index 8ba5bc35daf8c..952d7522b65f7 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ByteParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ByteParquetField.java @@ -12,13 +12,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for 8-bit signed byte values using {@link TinyIntVector}. */ -public class ByteParquetField extends ParquetField { +public class ByteParquetField extends NumericParquetField { /** Creates a new ByteParquetField. */ public ByteParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/DoubleParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/DoubleParquetField.java index ed45b0f083cd1..fef5ee3db63fe 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/DoubleParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/DoubleParquetField.java @@ -13,13 +13,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for double-precision floating-point values using {@link Float8Vector}. */ -public class DoubleParquetField extends ParquetField { +public class DoubleParquetField extends NumericParquetField { /** Creates a new DoubleParquetField. */ public DoubleParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/FloatParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/FloatParquetField.java index 280ceb0a6f75e..58ebc9906f3c8 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/FloatParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/FloatParquetField.java @@ -13,13 +13,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for single-precision floating-point values using {@link Float4Vector}. */ -public class FloatParquetField extends ParquetField { +public class FloatParquetField extends NumericParquetField { /** Creates a new FloatParquetField. */ public FloatParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/HalfFloatParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/HalfFloatParquetField.java index 09191c101e6f5..5d238ea8b056b 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/HalfFloatParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/HalfFloatParquetField.java @@ -13,13 +13,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for half-precision (16-bit) floating-point values using {@link Float2Vector}. */ -public class HalfFloatParquetField extends ParquetField { +public class HalfFloatParquetField extends NumericParquetField { /** Creates a new HalfFloatParquetField. */ public HalfFloatParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/IntegerParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/IntegerParquetField.java index 03e43d7da4901..96a86445b62a5 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/IntegerParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/IntegerParquetField.java @@ -12,13 +12,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for 32-bit signed integer values using {@link IntVector}. */ -public class IntegerParquetField extends ParquetField { +public class IntegerParquetField extends NumericParquetField { /** Creates a new IntegerParquetField. */ public IntegerParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/LongParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/LongParquetField.java index 7ef9d4b2fb243..27f0e4fe77c5a 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/LongParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/LongParquetField.java @@ -12,13 +12,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for 64-bit signed long values using {@link BigIntVector}. */ -public class LongParquetField extends ParquetField { +public class LongParquetField extends NumericParquetField { private final boolean nullable; diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/NumericParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/NumericParquetField.java new file mode 100644 index 0000000000000..fbf55ed681639 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/NumericParquetField.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.fields.core.data.number; + +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.parquet.fields.ParquetField; + +import java.util.Set; + +/** + * Parquet field for numeric values. Declares common capabilities supported for numeric field in parquet. + */ +public abstract class NumericParquetField extends ParquetField { + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.POINT_RANGE + ); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ShortParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ShortParquetField.java index cc9259afd6948..a75f7d3810f45 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ShortParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/ShortParquetField.java @@ -12,13 +12,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for 16-bit signed short values using {@link SmallIntVector}. */ -public class ShortParquetField extends ParquetField { +public class ShortParquetField extends NumericParquetField { /** Creates a new ShortParquetField. */ public ShortParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/TokenCountParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/TokenCountParquetField.java index 7a8cef044d9b4..bd2483eb8fd5c 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/TokenCountParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/TokenCountParquetField.java @@ -12,13 +12,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for token count values stored as 32-bit integers using {@link IntVector}. */ -public class TokenCountParquetField extends ParquetField { +public class TokenCountParquetField extends NumericParquetField { /** Creates a new TokenCountParquetField. */ public TokenCountParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/UnsignedLongParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/UnsignedLongParquetField.java index 79c7b3d8f1d04..4abadacacaa2c 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/UnsignedLongParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/number/UnsignedLongParquetField.java @@ -12,13 +12,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; /** * Parquet field for 64-bit unsigned long values using {@link UInt8Vector}. */ -public class UnsignedLongParquetField extends ParquetField { +public class UnsignedLongParquetField extends NumericParquetField { /** Creates a new UnsignedLongParquetField. */ public UnsignedLongParquetField() {} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/IpParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/IpParquetField.java index 940864ff0a15c..8815d822cbefb 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/IpParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/IpParquetField.java @@ -13,11 +13,13 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; import java.net.InetAddress; +import java.util.Set; /** * Parquet field for IP address values stored as binary using {@link VarBinaryVector}. @@ -43,4 +45,13 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.POINT_RANGE + ); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/TextParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/TextParquetField.java index 0805955c67bac..48124c636a8c0 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/TextParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/data/text/TextParquetField.java @@ -11,11 +11,13 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; import java.nio.charset.StandardCharsets; +import java.util.Set; /** * Parquet field for text values using {@link VarCharVector} with UTF-8 encoding. @@ -42,4 +44,13 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.STORED_FIELDS + ); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IdParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IdParquetField.java index 4675d23494e74..8618b52b56f5d 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IdParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IdParquetField.java @@ -11,10 +11,13 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; +import java.util.Set; + /** * Parquet field for document _id metadata stored as binary using {@link VarBinaryVector}. */ @@ -38,4 +41,13 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.notNullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.STORED_FIELDS + ); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IgnoredParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IgnoredParquetField.java index 5e39c27d4dc8d..b41e94d96a947 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IgnoredParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IgnoredParquetField.java @@ -11,11 +11,16 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; import java.nio.charset.StandardCharsets; +import java.util.Set; + +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.STORED_FIELDS; /** * Parquet field for _ignored metadata stored as UTF-8 using {@link VarCharVector}. @@ -42,4 +47,9 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of(STORED_FIELDS, COLUMNAR_STORAGE); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IndexParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IndexParquetField.java new file mode 100644 index 0000000000000..522615cefdda1 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/IndexParquetField.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.fields.core.metadata; + +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.parquet.fields.ParquetField; +import org.opensearch.parquet.vsr.ManagedVSR; + +/** + * Parquet field for index name. + * This is a placeholder for declaring support for the field. + */ +public class IndexParquetField extends ParquetField { + + @Override + protected void addToGroup(MappedFieldType fieldType, ManagedVSR managedVSR, Object parseValue) { + throw new IllegalStateException("Index field is not supposed to be added to group"); + } + + @Override + public ArrowType getArrowType() { + return null; + } + + @Override + public FieldType getFieldType() { + return null; + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/RoutingParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/RoutingParquetField.java index db2048eaf7594..49a102a046f0c 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/RoutingParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/core/metadata/RoutingParquetField.java @@ -11,11 +11,13 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.vsr.ManagedVSR; import java.nio.charset.StandardCharsets; +import java.util.Set; /** * Parquet field for _routing metadata stored as UTF-8 using {@link VarCharVector}. @@ -42,4 +44,13 @@ public ArrowType getArrowType() { public FieldType getFieldType() { return FieldType.nullable(getArrowType()); } + + @Override + public Set supportedCapabilities() { + return Set.of( + FieldTypeCapabilities.Capability.COLUMNAR_STORAGE, + FieldTypeCapabilities.Capability.BLOOM_FILTER, + FieldTypeCapabilities.Capability.STORED_FIELDS + ); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/plugins/MetadataFieldPlugin.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/plugins/MetadataFieldPlugin.java index ea00e9e4d8488..c3ebf95d17f77 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/plugins/MetadataFieldPlugin.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/plugins/MetadataFieldPlugin.java @@ -8,21 +8,30 @@ package org.opensearch.parquet.fields.plugins; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.IgnoredFieldMapper; +import org.opensearch.index.mapper.IndexFieldMapper; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.parquet.fields.ParquetField; +import org.opensearch.parquet.fields.core.data.BinaryParquetField; import org.opensearch.parquet.fields.core.data.number.IntegerParquetField; import org.opensearch.parquet.fields.core.data.number.LongParquetField; import org.opensearch.parquet.fields.core.metadata.IdParquetField; import org.opensearch.parquet.fields.core.metadata.IgnoredParquetField; +import org.opensearch.parquet.fields.core.metadata.IndexParquetField; import org.opensearch.parquet.fields.core.metadata.RoutingParquetField; import java.util.HashMap; import java.util.Map; +import java.util.Set; + +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.STORED_FIELDS; /** * Metadata fields plugin providing Parquet field implementations for OpenSearch metadata fields. @@ -37,12 +46,19 @@ public Map getParquetFields() { final Map fieldMap = new HashMap<>(); fieldMap.put(DocCountFieldMapper.CONTENT_TYPE, new LongParquetField()); fieldMap.put("_size", new IntegerParquetField()); + fieldMap.put(IndexFieldMapper.CONTENT_TYPE, new IndexParquetField()); fieldMap.put(RoutingFieldMapper.CONTENT_TYPE, new RoutingParquetField()); fieldMap.put(IgnoredFieldMapper.CONTENT_TYPE, new IgnoredParquetField()); fieldMap.put(IdFieldMapper.CONTENT_TYPE, new IdParquetField()); fieldMap.put(SeqNoFieldMapper.CONTENT_TYPE, new LongParquetField(false)); fieldMap.put(SeqNoFieldMapper.PRIMARY_TERM_NAME, new LongParquetField(false)); fieldMap.put(VersionFieldMapper.CONTENT_TYPE, new LongParquetField(false)); + fieldMap.put(SourceFieldMapper.NAME, new BinaryParquetField() { + @Override + public Set supportedCapabilities() { + return Set.of(STORED_FIELDS, COLUMNAR_STORAGE); + } + }); return fieldMap; } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetDocumentInput.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetDocumentInput.java index 8a01726165fd4..7832b4f990f90 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetDocumentInput.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetDocumentInput.java @@ -8,14 +8,20 @@ package org.opensearch.parquet.writer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.index.engine.dataformat.DocumentInput; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.engine.exec.PrimaryTermFieldType; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.VersionFieldMapper; +import org.opensearch.parquet.ParquetDataFormatPlugin; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * Document input for the Parquet data format. @@ -29,16 +35,21 @@ */ public class ParquetDocumentInput implements DocumentInput> { + private static final Logger logger = LogManager.getLogger(ParquetDocumentInput.class); private final List collectedFields = new ArrayList<>(); private long rowId = -1; private boolean isClosed = false; - /** Creates a new ParquetDocumentInput. */ - public ParquetDocumentInput() {} - @Override public void addField(MappedFieldType fieldType, Object value) { ensureOpen(); + Set capabilities = fieldType.getCapabilityMap() + .getOrDefault(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of()); + if (capabilities.isEmpty() && fieldType != PrimaryTermFieldType.INSTANCE) { + // nothing to support on this format for this field. + logger.trace("Ignored to add field: {} {}", fieldType.name(), fieldType.getCapabilityMap()); + return; + } collectedFields.add(new FieldValuePair(fieldType, value)); } diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetBaseTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetBaseTests.java new file mode 100644 index 0000000000000..fc408d26baa4b --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetBaseTests.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet; + +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.lucene.search.Query; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatTestUtils; +import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.TextSearchInfo; +import org.opensearch.index.mapper.ValueFetcher; +import org.opensearch.index.mapper.VersionFieldMapper; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.parquet.writer.ParquetDocumentInput; +import org.opensearch.search.lookup.SearchLookup; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; + +public abstract class ParquetBaseTests extends OpenSearchTestCase { + + public static final MappedFieldType SEQ_NO_FIELD = new NumberFieldMapper.NumberFieldType( + SeqNoFieldMapper.NAME, + NumberFieldMapper.NumberType.LONG + ); + + public static final MappedFieldType PRIMARY_TERM_FIELD = new NumberFieldMapper.NumberFieldType( + SeqNoFieldMapper.PRIMARY_TERM_NAME, + NumberFieldMapper.NumberType.LONG + ); + + public static final MappedFieldType VERSION_FIELD = new NumberFieldMapper.NumberFieldType( + VersionFieldMapper.NAME, + NumberFieldMapper.NumberType.LONG + ); + + public static final MappedFieldType ID_FIELD = new MappedFieldType( + IdFieldMapper.CONTENT_TYPE, + true, + true, + true, + TextSearchInfo.SIMPLE_MATCH_ONLY, + Map.of() + ) { + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return IdFieldMapper.CONTENT_TYPE; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + }; + + static { + SEQ_NO_FIELD.setCapabilityMap(Map.of(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of(COLUMNAR_STORAGE))); + VERSION_FIELD.setCapabilityMap(Map.of(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of(COLUMNAR_STORAGE))); + ID_FIELD.setCapabilityMap(Map.of(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of(COLUMNAR_STORAGE))); + PRIMARY_TERM_FIELD.setCapabilityMap(Map.of(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of(COLUMNAR_STORAGE))); + } + + public static List metadataFields() { + List fields = new ArrayList<>(); + fields.add(new Field(VersionFieldMapper.NAME, FieldType.notNullable(new ArrowType.Int(64, true)), null)); + fields.add(new Field(SeqNoFieldMapper.NAME, FieldType.notNullable(new ArrowType.Int(64, true)), null)); + fields.add(new Field(SeqNoFieldMapper.PRIMARY_TERM_NAME, FieldType.notNullable(new ArrowType.Int(64, true)), null)); + fields.add(new Field(IdFieldMapper.NAME, FieldType.notNullable(new ArrowType.Binary()), null)); + return fields; + } + + protected void populateMetadataFields(ParquetDocumentInput input) { + input.addField(SEQ_NO_FIELD, 100L); + input.addField(ID_FIELD, "id".getBytes(StandardCharsets.UTF_8)); + input.addField(VERSION_FIELD, 1L); + input.addField(PRIMARY_TERM_FIELD, 1L); + } + + protected void assignTestCapabilities(MappedFieldType fieldType, DataFormat format) { + DataFormatTestUtils.assignTestCapabilities(fieldType, format); + } + + protected NumberFieldMapper.NumberFieldType createNumberField(String name, NumberFieldMapper.NumberType type) { + NumberFieldMapper.NumberFieldType numberFieldType = new NumberFieldMapper.NumberFieldType(name, type); + assignTestCapabilities(numberFieldType, ParquetDataFormatPlugin.PARQUET_DATA_FORMAT); + return numberFieldType; + } + + protected KeywordFieldMapper.KeywordFieldType createKeywordField(String name) { + KeywordFieldMapper.KeywordFieldType keywordFieldType = new KeywordFieldMapper.KeywordFieldType(name); + assignTestCapabilities(keywordFieldType, ParquetDataFormatPlugin.PARQUET_DATA_FORMAT); + return keywordFieldType; + } +} diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatAwareEngineTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatAwareEngineTests.java index 609a3a9e6c4c9..1f103b508a461 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatAwareEngineTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatAwareEngineTests.java @@ -12,7 +12,6 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.lucene.search.Query; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; @@ -27,10 +26,10 @@ import org.opensearch.index.mapper.BooleanFieldMapper.BooleanFieldType; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.DateFieldMapper.DateFieldType; -import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.IpFieldMapper.IpFieldType; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MatchOnlyTextFieldMapper; import org.opensearch.index.mapper.MatchOnlyTextFieldMapper.MatchOnlyTextFieldType; import org.opensearch.index.mapper.NumberFieldMapper; @@ -38,9 +37,7 @@ import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.TextFieldMapper.TextFieldType; import org.opensearch.index.mapper.TextSearchInfo; -import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.mapper.VersionFieldMapper; -import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.PrecomputedChecksumStrategy; import org.opensearch.index.store.Store; @@ -51,7 +48,6 @@ import org.opensearch.parquet.fields.plugins.CoreDataFieldPlugin; import org.opensearch.parquet.writer.ParquetDocumentInput; import org.opensearch.plugins.SearchBackEndPlugin; -import org.opensearch.search.lookup.SearchLookup; import org.opensearch.test.IndexSettingsModule; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.TestThreadPool; @@ -63,8 +59,16 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; -import static org.opensearch.parquet.engine.ParquetIndexingEngineTests.metadataFields; +import static org.opensearch.index.engine.dataformat.DataFormatTestUtils.assignTestCapabilities; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.parquet.ParquetBaseTests.ID_FIELD; +import static org.opensearch.parquet.ParquetBaseTests.SEQ_NO_FIELD; +import static org.opensearch.parquet.ParquetBaseTests.VERSION_FIELD; +import static org.opensearch.parquet.ParquetBaseTests.metadataFields; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Runs the {@link AbstractDataFormatAwareEngineTestCase} suite with the real @@ -82,37 +86,11 @@ public class ParquetDataFormatAwareEngineTests extends AbstractDataFormatAwareEn private static final MappedFieldType NAME_FIELD = new KeywordFieldMapper.KeywordFieldType("name"); private static final MappedFieldType AGE_FIELD = new NumberFieldMapper.NumberFieldType("age", NumberFieldMapper.NumberType.INTEGER); - public static final MappedFieldType SEQ_NO_FIELD = new NumberFieldMapper.NumberFieldType( - SeqNoFieldMapper.NAME, - NumberFieldMapper.NumberType.LONG - ); - public static final MappedFieldType VERSION_FIELD = new NumberFieldMapper.NumberFieldType( - VersionFieldMapper.NAME, - NumberFieldMapper.NumberType.LONG - ); - public static final MappedFieldType ID_FIELD = new MappedFieldType( - IdFieldMapper.CONTENT_TYPE, - true, - true, - true, - TextSearchInfo.SIMPLE_MATCH_ONLY, - Map.of() - ) { - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - return null; - } - - @Override - public String typeName() { - return IdFieldMapper.CONTENT_TYPE; - } - @Override - public Query termQuery(Object value, QueryShardContext context) { - return null; - } - }; + static { + NAME_FIELD.setCapabilityMap(Map.of(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of(COLUMNAR_STORAGE))); + AGE_FIELD.setCapabilityMap(Map.of(ParquetDataFormatPlugin.PARQUET_DATA_FORMAT, Set.of(COLUMNAR_STORAGE))); + } private Schema schema; private org.opensearch.arrow.allocator.ArrowNativeAllocator nativeAllocator; @@ -214,33 +192,84 @@ protected String dataFormatName() { @Override protected DocumentInput createDocumentInput() { + ParquetDataFormat format = new ParquetDataFormat(); ParquetDocumentInput input = new ParquetDocumentInput(); input.addField(ID_FIELD, "doc-id".getBytes(StandardCharsets.UTF_8)); input.addField(NAME_FIELD, "name"); - input.addField(new NumberFieldMapper.NumberFieldType(BYTE_FIELD_NAME, NumberFieldMapper.NumberType.BYTE), Byte.MAX_VALUE); - input.addField(new NumberFieldMapper.NumberFieldType(SHORT_FIELD_NAME, NumberFieldMapper.NumberType.SHORT), Short.MAX_VALUE); - input.addField(new NumberFieldMapper.NumberFieldType(INT_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER), Integer.MAX_VALUE); - input.addField(new NumberFieldMapper.NumberFieldType(LONG_FIELD_NAME, NumberFieldMapper.NumberType.LONG), Long.MAX_VALUE); - input.addField(new NumberFieldMapper.NumberFieldType(FLOAT_FIELD_NAME, NumberFieldMapper.NumberType.FLOAT), Float.MAX_VALUE); - input.addField(new NumberFieldMapper.NumberFieldType(DOUBLE_FIELD_NAME, NumberFieldMapper.NumberType.DOUBLE), Double.MAX_VALUE); - input.addField( + addFieldWithCapabilities( + input, + new NumberFieldMapper.NumberFieldType(BYTE_FIELD_NAME, NumberFieldMapper.NumberType.BYTE), + Byte.MAX_VALUE, + format + ); + addFieldWithCapabilities( + input, + new NumberFieldMapper.NumberFieldType(SHORT_FIELD_NAME, NumberFieldMapper.NumberType.SHORT), + Short.MAX_VALUE, + format + ); + addFieldWithCapabilities( + input, + new NumberFieldMapper.NumberFieldType(INT_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER), + Integer.MAX_VALUE, + format + ); + addFieldWithCapabilities( + input, + new NumberFieldMapper.NumberFieldType(LONG_FIELD_NAME, NumberFieldMapper.NumberType.LONG), + Long.MAX_VALUE, + format + ); + addFieldWithCapabilities( + input, + new NumberFieldMapper.NumberFieldType(FLOAT_FIELD_NAME, NumberFieldMapper.NumberType.FLOAT), + Float.MAX_VALUE, + format + ); + addFieldWithCapabilities( + input, + new NumberFieldMapper.NumberFieldType(DOUBLE_FIELD_NAME, NumberFieldMapper.NumberType.DOUBLE), + Double.MAX_VALUE, + format + ); + addFieldWithCapabilities( + input, new NumberFieldMapper.NumberFieldType(HALF_FLOAT_FIELD_NAME, NumberFieldMapper.NumberType.HALF_FLOAT), - Short.MAX_VALUE + Short.MAX_VALUE, + format ); - input.addField( + addFieldWithCapabilities( + input, new NumberFieldMapper.NumberFieldType(UNSIGNED_LONG_FIELD_NAME, NumberFieldMapper.NumberType.UNSIGNED_LONG), - Long.MAX_VALUE + Long.MAX_VALUE, + format + ); + addFieldWithCapabilities(input, new TextFieldType(TEXT_FIELD_NAME), randomAlphaOfLength(100), format); + addFieldWithCapabilities(input, new DateFieldType(DATE_FIELD_NAME), System.currentTimeMillis(), format); + addFieldWithCapabilities( + input, + new DateFieldType(DATE_NANOS_FIELD_NAME, DateFieldMapper.Resolution.NANOSECONDS), + System.nanoTime(), + format ); - input.addField(new TextFieldType(TEXT_FIELD_NAME), randomAlphaOfLength(100)); - input.addField(new DateFieldType(DATE_FIELD_NAME), System.currentTimeMillis()); - input.addField(new DateFieldType(DATE_NANOS_FIELD_NAME, DateFieldMapper.Resolution.NANOSECONDS), System.nanoTime()); - input.addField(new IpFieldType(IP_FIELD_NAME), InetAddresses.forString("0.0.0.0")); - input.addField(new BinaryFieldType(BINARY_FIELD_NAME), randomAlphaOfLength(100).getBytes(StandardCharsets.UTF_8)); - input.addField(new BooleanFieldType(BOOLEAN_FIELD_NAME), randomBoolean()); + addFieldWithCapabilities(input, new IpFieldType(IP_FIELD_NAME), InetAddresses.forString("0.0.0.0"), format); + addFieldWithCapabilities( + input, + new BinaryFieldType(BINARY_FIELD_NAME), + randomAlphaOfLength(100).getBytes(StandardCharsets.UTF_8), + format + ); + addFieldWithCapabilities(input, new BooleanFieldType(BOOLEAN_FIELD_NAME), randomBoolean(), format); + assignTestCapabilities(matchOnlyTextFieldType, format); input.addField(matchOnlyTextFieldType, randomAlphaOfLength(100)); return input; } + private void addFieldWithCapabilities(ParquetDocumentInput input, MappedFieldType fieldType, Object value, ParquetDataFormat format) { + assignTestCapabilities(fieldType, format); + input.addField(fieldType, value); + } + private static final String BYTE_FIELD_NAME = "byte_field"; private static final String SHORT_FIELD_NAME = "short_field"; private static final String INT_FIELD_NAME = "integer_field"; @@ -278,4 +307,16 @@ private Schema buildSchema() { fields.add(new Field(DocumentInput.ROW_ID_FIELD, FieldType.notNullable(new ArrowType.Int(64, true)), null)); return new Schema(fields); } + + @Override + protected MapperService createMockMapperService(IndexSettings indexSettings) { + MapperService ms = mock(MapperService.class); + when(ms.fieldType(VersionFieldMapper.NAME)).thenReturn(VERSION_FIELD); + when(ms.fieldType(SeqNoFieldMapper.NAME)).thenReturn(SEQ_NO_FIELD); + when(ms.getIndexSettings()).thenReturn(indexSettings); + org.opensearch.index.mapper.DocumentMapper documentMapper = mock(org.opensearch.index.mapper.DocumentMapper.class); + when(documentMapper.getVersion()).thenReturn(1L); + when(ms.documentMapper()).thenReturn(documentMapper); + return ms; + } } diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatTests.java index 343518cc30c7e..b7f588b00ce5c 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetDataFormatTests.java @@ -25,7 +25,10 @@ public void testSupportedFieldsContainsColumnarStorage() { var fields = new ParquetDataFormat().supportedFields(); assertFalse(fields.isEmpty()); for (var ftc : fields) { - assertTrue(ftc.capabilities().contains(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE)); + assertTrue( + "filed: " + ftc + " should have columnar support", + ftc.capabilities().contains(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE) + ); } } } diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java index 54c783e2ac6de..c08def804ebd6 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java @@ -8,9 +8,7 @@ package org.opensearch.parquet.engine; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.opensearch.Version; import org.opensearch.arrow.allocator.ArrowNativeAllocator; @@ -25,38 +23,29 @@ import org.opensearch.index.engine.dataformat.RefreshResult; import org.opensearch.index.engine.dataformat.Writer; import org.opensearch.index.engine.dataformat.WriterConfig; -import org.opensearch.index.engine.exec.PrimaryTermFieldType; -import org.opensearch.index.mapper.IdFieldMapper; -import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.index.shard.ShardPath; +import org.opensearch.parquet.ParquetBaseTests; import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.parquet.fields.ArrowFieldRegistry; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.writer.ParquetDocumentInput; -import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.ThreadPool; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; -import static org.opensearch.parquet.engine.ParquetDataFormatAwareEngineTests.ID_FIELD; -import static org.opensearch.parquet.engine.ParquetDataFormatAwareEngineTests.SEQ_NO_FIELD; -import static org.opensearch.parquet.engine.ParquetDataFormatAwareEngineTests.VERSION_FIELD; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class ParquetIndexingEngineTests extends OpenSearchTestCase { +public class ParquetIndexingEngineTests extends ParquetBaseTests { private org.opensearch.arrow.allocator.ArrowNativeAllocator nativeAllocator; private MappedFieldType idField; @@ -73,9 +62,11 @@ public void setUp() throws Exception { RustBridge.initLogger(); nativeAllocator = new ArrowNativeAllocator(Long.MAX_VALUE); nativeAllocator.getOrCreatePool(NativeAllocatorPoolConfig.POOL_INGEST, 0L, Long.MAX_VALUE); - idField = new NumberFieldMapper.NumberFieldType("id", NumberFieldMapper.NumberType.INTEGER); - nameField = new KeywordFieldMapper.KeywordFieldType("name"); - scoreField = new NumberFieldMapper.NumberFieldType("score", NumberFieldMapper.NumberType.LONG); + + idField = createNumberField("id", NumberFieldMapper.NumberType.INTEGER); + nameField = createKeywordField("name"); + scoreField = createNumberField("score", NumberFieldMapper.NumberType.LONG); + schema = buildSchema(List.of(idField, nameField, scoreField)); tempDir = createTempDir(); Settings settings = Settings.builder().put("node.name", "parquetengine-test").build(); @@ -251,20 +242,4 @@ private MapperService createMockMapperService(Schema schema, IndexSettings index when(mapperService.getIndexSettings()).thenReturn(indexSettings); return mapperService; } - - public static List metadataFields() { - List fields = new ArrayList<>(); - fields.add(new Field(VersionFieldMapper.NAME, FieldType.notNullable(new ArrowType.Int(64, true)), null)); - fields.add(new Field(SeqNoFieldMapper.NAME, FieldType.notNullable(new ArrowType.Int(64, true)), null)); - fields.add(new Field(SeqNoFieldMapper.PRIMARY_TERM_NAME, FieldType.notNullable(new ArrowType.Int(64, true)), null)); - fields.add(new Field(IdFieldMapper.NAME, FieldType.notNullable(new ArrowType.Binary()), null)); - return fields; - } - - public static void populateMetadataFields(ParquetDocumentInput input) { - input.addField(SEQ_NO_FIELD, 100L); - input.addField(ID_FIELD, "id".getBytes(StandardCharsets.UTF_8)); - input.addField(VERSION_FIELD, 1L); - input.addField(PrimaryTermFieldType.INSTANCE, 1L); - } } diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/fields/plugins/MetadataFieldPluginTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/fields/plugins/MetadataFieldPluginTests.java index 046c39efa55f8..d94ab26c5012e 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/fields/plugins/MetadataFieldPluginTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/fields/plugins/MetadataFieldPluginTests.java @@ -11,8 +11,10 @@ import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.IgnoredFieldMapper; +import org.opensearch.index.mapper.IndexFieldMapper; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.test.OpenSearchTestCase; @@ -30,7 +32,7 @@ public void setUp() throws Exception { } public void testFieldCount() { - assertEquals(8, fields.size()); + assertEquals("expected 10 fields but got: " + fields.keySet(), 10, fields.size()); } public void testAllMetadataTypesPresent() { @@ -42,6 +44,8 @@ public void testAllMetadataTypesPresent() { assertNotNull(fields.get(SeqNoFieldMapper.CONTENT_TYPE)); assertNotNull(fields.get(SeqNoFieldMapper.PRIMARY_TERM_NAME)); assertNotNull(fields.get(VersionFieldMapper.CONTENT_TYPE)); + assertNotNull(fields.get(IndexFieldMapper.CONTENT_TYPE)); + assertNotNull(fields.get(SourceFieldMapper.CONTENT_TYPE)); } public void testAllValuesNonNull() { diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java index c9584463dc6ad..d34e1a24f0457 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java @@ -18,15 +18,17 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.parquet.ParquetBaseTests; import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.bridge.ParquetFileMetadata; import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.memory.ArrowBufferPool; import org.opensearch.parquet.writer.ParquetDocumentInput; -import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.ThreadPool; @@ -34,11 +36,9 @@ import java.util.List; import java.util.concurrent.Future; -import static org.opensearch.parquet.engine.ParquetIndexingEngineTests.metadataFields; -import static org.opensearch.parquet.engine.ParquetIndexingEngineTests.populateMetadataFields; - -public class VSRManagerTests extends OpenSearchTestCase { +public class VSRManagerTests extends ParquetBaseTests { + private static final DataFormat PARQUET_FORMAT = new ParquetDataFormat(); private ArrowNativeAllocator nativeAllocator; private ArrowBufferPool bufferPool; /** Minimal schema VSRManager is constructed with; addDocument tests reconcile metadata fields in via {@link #reconcileMetadata}. */ @@ -128,6 +128,7 @@ public void testAddDocument() throws Exception { VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L); NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); + assignTestCapabilities(valField, PARQUET_FORMAT); ParquetDocumentInput doc = new ParquetDocumentInput(); populateMetadataFields(doc); doc.addField(valField, 42); @@ -320,6 +321,8 @@ public void testAddDocumentAfterReconcileSchemaAddsVector() throws Exception { NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); KeywordFieldMapper.KeywordFieldType tagField = new KeywordFieldMapper.KeywordFieldType("tag"); + assignTestCapabilities(valField, PARQUET_FORMAT); + assignTestCapabilities(tagField, PARQUET_FORMAT); ParquetDocumentInput doc = new ParquetDocumentInput(); populateMetadataFields(doc); doc.setRowId(DocumentInput.ROW_ID_FIELD, 0); @@ -340,6 +343,7 @@ public void testIsSchemaMutableBeforeAndAfterFlush() throws Exception { assertTrue(manager.isSchemaMutable()); NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); + assignTestCapabilities(valField, PARQUET_FORMAT); ParquetDocumentInput doc = new ParquetDocumentInput(); populateMetadataFields(doc); doc.setRowId(DocumentInput.ROW_ID_FIELD, 0); @@ -356,6 +360,8 @@ public void testSchemaUpdatePropagatesAcrossRotation() throws Exception { NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); KeywordFieldMapper.KeywordFieldType tagField = new KeywordFieldMapper.KeywordFieldType("tag"); + assignTestCapabilities(valField, PARQUET_FORMAT); + assignTestCapabilities(tagField, PARQUET_FORMAT); // Reconcile once before any docs — the tag vector must persist across the VSR // rotation triggered by maxRowsPerVSR=1. @@ -408,6 +414,10 @@ public void testReconcileSchemaAddsMultipleVectorsAtOnce() throws Exception { KeywordFieldMapper.KeywordFieldType tag1Field = new KeywordFieldMapper.KeywordFieldType("tag1"); KeywordFieldMapper.KeywordFieldType tag2Field = new KeywordFieldMapper.KeywordFieldType("tag2"); KeywordFieldMapper.KeywordFieldType tag3Field = new KeywordFieldMapper.KeywordFieldType("tag3"); + assignTestCapabilities(valField, PARQUET_FORMAT); + assignTestCapabilities(tag1Field, PARQUET_FORMAT); + assignTestCapabilities(tag2Field, PARQUET_FORMAT); + assignTestCapabilities(tag3Field, PARQUET_FORMAT); ParquetDocumentInput doc = new ParquetDocumentInput(); populateMetadataFields(doc); @@ -522,7 +532,7 @@ public void testAddDocumentAfterSuccessfulBackgroundWriteDoesNotThrow() throws E int lowThreshold = randomIntBetween(2, 5); VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, lowThreshold, threadPool, 0L); - NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); + NumberFieldMapper.NumberFieldType valField = createNumberField("val", NumberFieldMapper.NumberType.INTEGER); // Run multiple rotation cycles — each cycle fills the VSR to threshold, // triggers background write, waits for completion, then verifies next addDocument works @@ -570,7 +580,7 @@ public void testContinuousAddDocumentAcrossMultipleRotationsWithoutWaiting() thr int totalDocs = lowThreshold * randomIntBetween(5, 12); VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, lowThreshold, threadPool, 0L); - NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); + NumberFieldMapper.NumberFieldType valField = createNumberField("val", NumberFieldMapper.NumberType.INTEGER); // Add all docs in a tight loop — no waiting between rotations for (int i = 0; i < totalDocs; i++) { diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetDocumentInputTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetDocumentInputTests.java index f0bd5e5a2e5ed..8fb9f82a3ae2d 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetDocumentInputTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetDocumentInputTests.java @@ -8,22 +8,26 @@ package org.opensearch.parquet.writer; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.parquet.ParquetBaseTests; +import org.opensearch.parquet.engine.ParquetDataFormat; import java.util.List; -import static org.opensearch.parquet.engine.ParquetIndexingEngineTests.populateMetadataFields; +public class ParquetDocumentInputTests extends ParquetBaseTests { -public class ParquetDocumentInputTests extends OpenSearchTestCase { + private static final DataFormat PARQUET_FORMAT = new ParquetDataFormat(); public void testAddFieldAndGetFinalInput() { ParquetDocumentInput input = new ParquetDocumentInput(); MappedFieldType ft = new NumberFieldMapper.NumberFieldType("age", NumberFieldMapper.NumberType.INTEGER); + assignTestCapabilities(ft, PARQUET_FORMAT); input.addField(ft, 25); - input.setRowId("__row_id__", 0L); + input.setRowId(DocumentInput.ROW_ID_FIELD, 0L); populateMetadataFields(input); List result = input.getFinalInput(); assertEquals(5, result.size()); @@ -35,9 +39,11 @@ public void testMultipleFields() { ParquetDocumentInput input = new ParquetDocumentInput(); MappedFieldType ft1 = new NumberFieldMapper.NumberFieldType("a", NumberFieldMapper.NumberType.INTEGER); MappedFieldType ft2 = new KeywordFieldMapper.KeywordFieldType("b"); + assignTestCapabilities(ft1, PARQUET_FORMAT); + assignTestCapabilities(ft2, PARQUET_FORMAT); input.addField(ft1, 1); input.addField(ft2, "val"); - input.setRowId("__row_id__", 0L); + input.setRowId(DocumentInput.ROW_ID_FIELD, 0L); populateMetadataFields(input); assertEquals(6, input.getFinalInput().size()); } @@ -45,29 +51,24 @@ public void testMultipleFields() { public void testEmptyInput() { ParquetDocumentInput input = new ParquetDocumentInput(); populateMetadataFields(input); - input.setRowId("__row_id__", 0L); + input.setRowId(DocumentInput.ROW_ID_FIELD, 0L); assertEquals(4, input.getFinalInput().size()); } public void testSetRowId() { ParquetDocumentInput input = new ParquetDocumentInput(); populateMetadataFields(input); - input.setRowId("__row_id__", 42L); + input.setRowId(DocumentInput.ROW_ID_FIELD, 42L); assertEquals(42L, input.getRowId()); } - public void testAddFieldRejectsNullFieldType() { - ParquetDocumentInput input = new ParquetDocumentInput(); - populateMetadataFields(input); - expectThrows(IllegalArgumentException.class, () -> input.addField(null, "ignored")); - } - public void testCloseClearsState() { ParquetDocumentInput input = new ParquetDocumentInput(); populateMetadataFields(input); MappedFieldType ft = new NumberFieldMapper.NumberFieldType("age", NumberFieldMapper.NumberType.INTEGER); + assignTestCapabilities(ft, PARQUET_FORMAT); input.addField(ft, 25); - input.setRowId("__row_id__", 0L); + input.setRowId(DocumentInput.ROW_ID_FIELD, 0L); assertEquals(5, input.getFinalInput().size()); input.close(); diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java index 09bf28a908441..f1cb2a59ad270 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java @@ -22,13 +22,13 @@ import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.parquet.ParquetBaseTests; import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.fields.ArrowFieldRegistry; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.memory.ArrowBufferPool; -import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.ThreadPool; @@ -37,11 +37,9 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.parquet.engine.ParquetIndexingEngineTests.metadataFields; -import static org.opensearch.parquet.engine.ParquetIndexingEngineTests.populateMetadataFields; - -public class ParquetWriterTests extends OpenSearchTestCase { +public class ParquetWriterTests extends ParquetBaseTests { + private final ParquetDataFormat parquetFormat = new ParquetDataFormat(); private ArrowNativeAllocator nativeAllocator; private ArrowBufferPool bufferPool; private MappedFieldType idField; @@ -61,6 +59,9 @@ public void setUp() throws Exception { idField = new NumberFieldMapper.NumberFieldType("id", NumberFieldMapper.NumberType.INTEGER); nameField = new KeywordFieldMapper.KeywordFieldType("name"); scoreField = new NumberFieldMapper.NumberFieldType("score", NumberFieldMapper.NumberType.LONG); + assignTestCapabilities(idField, parquetFormat); + assignTestCapabilities(nameField, parquetFormat); + assignTestCapabilities(scoreField, parquetFormat); schema = buildSchema(List.of(idField, nameField, scoreField)); Settings indexSettingsBuilder = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/AnalyticsQueryTaskCleanupIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/AnalyticsQueryTaskCleanupIT.java index 9edce1c646ff5..221fdaa1bcc6e 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/AnalyticsQueryTaskCleanupIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/AnalyticsQueryTaskCleanupIT.java @@ -26,7 +26,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -96,7 +96,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/ReduceThreadPoolCleanupIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/ReduceThreadPoolCleanupIT.java index 4337e8c928624..7e7c2eb577727 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/ReduceThreadPoolCleanupIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/ReduceThreadPoolCleanupIT.java @@ -26,7 +26,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -93,7 +93,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/SearchCancellationIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/SearchCancellationIT.java index ea4892f4b64bc..497340fe00d71 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/SearchCancellationIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/cancellation/SearchCancellationIT.java @@ -28,7 +28,6 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -36,6 +35,7 @@ import org.opensearch.ppl.action.PPLResponse; import org.opensearch.ppl.action.UnifiedPPLExecuteAction; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.TransportService; @@ -95,7 +95,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java index 1b8c71b6517ef..f46de7df3b550 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java @@ -39,7 +39,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.core.transport.TransportResponse; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -157,7 +157,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTopologyTestBase.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTopologyTestBase.java index 3e0da8d930d1c..8fa20072a87c7 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTopologyTestBase.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTopologyTestBase.java @@ -21,7 +21,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -67,7 +67,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTransportStressIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTransportStressIT.java index fbb466fc2ebaa..609d802aea052 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTransportStressIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorTransportStressIT.java @@ -35,7 +35,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -163,7 +163,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MaxShardsPerQueryIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MaxShardsPerQueryIT.java index 9118c33a10e46..97368fd327194 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MaxShardsPerQueryIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MaxShardsPerQueryIT.java @@ -24,7 +24,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.test.OpenSearchIntegTestCase; @@ -55,7 +55,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MemoryGuardIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MemoryGuardIT.java index 3ae1f328f1a77..0387954e75e7a 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MemoryGuardIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/MemoryGuardIT.java @@ -26,7 +26,7 @@ import org.opensearch.core.indices.breaker.CircuitBreakerStats; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -70,7 +70,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } @@ -104,7 +104,7 @@ private void createIndexAndIngest() throws Exception { .startObject() .startObject("properties") .startObject("user_id").field("type", "long").endObject() - .startObject("url").field("type", "keyword").endObject() + .startObject("url").field("type", "keyword").field("index", "false").endObject() .startObject("count").field("type", "integer").endObject() .endObject() .endObject(); diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceEarlyTerminationIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceEarlyTerminationIT.java index 245db3f0cd725..fce808338a6c6 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceEarlyTerminationIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceEarlyTerminationIT.java @@ -22,7 +22,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -78,7 +78,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } @@ -125,7 +125,7 @@ private void createAndSeedIndex() throws Exception { .indices() .prepareCreate(INDEX) .setSettings(indexSettings) - .setMapping("category", "type=keyword", "value", "type=integer") + .setMapping("category", "type=keyword,index=false", "value", "type=integer") .get() .isAcknowledged() ); @@ -268,7 +268,7 @@ private void createAndSeedLargeIndex() throws Exception { .indices() .prepareCreate(LARGE_INDEX) .setSettings(indexSettings) - .setMapping("category", "type=keyword", "value", "type=integer") + .setMapping("category", "type=keyword,index=false", "value", "type=integer") .get() .isAcknowledged() ); diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceTargetPartitionsIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceTargetPartitionsIT.java index 3607c393fdd00..c9ba00a26f3b9 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceTargetPartitionsIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ReduceTargetPartitionsIT.java @@ -21,7 +21,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -65,7 +65,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ShardFailoverIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ShardFailoverIT.java index b0aa901d11f5b..2b65c9f21420c 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ShardFailoverIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/ShardFailoverIT.java @@ -25,7 +25,7 @@ import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -73,7 +73,7 @@ protected Collection> nodePlugins() { super.nodePlugins().stream(), Stream.of( ArrowBasePlugin.class, - ParquetDataFormatPlugin.class, + ParquetOnlyDataFormatPlugin.class, CompositeDataFormatPlugin.class, MockCommitterEnginePlugin.class, MockTransportService.TestPlugin.class, diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/SpillDisabledParallelismIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/SpillDisabledParallelismIT.java index 9e58bf32d1ea0..18ba15986a713 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/SpillDisabledParallelismIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/SpillDisabledParallelismIT.java @@ -20,7 +20,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -72,7 +72,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/AnalyticsSearchSlowLogIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/AnalyticsSearchSlowLogIT.java index 1e1ffaabc0b26..77d36a93c3d09 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/AnalyticsSearchSlowLogIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/AnalyticsSearchSlowLogIT.java @@ -31,7 +31,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.ppl.TestPPLPlugin; @@ -68,7 +68,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/ValuesSqlIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/ValuesSqlIT.java index 1d9c26395365f..81ae1c7b9cd8a 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/ValuesSqlIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/ValuesSqlIT.java @@ -21,7 +21,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.test.OpenSearchIntegTestCase; @@ -55,7 +55,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } @@ -228,7 +228,7 @@ private void createAndSeedHttpLogsIndex(int shardCount) { .indices() .prepareCreate("http_logs") .setSettings(indexSettings) - .setMapping("verb", "type=keyword", "size", "type=integer") + .setMapping("verb", "type=keyword,index=false", "size", "type=integer") .get(); assertTrue("http_logs creation must be acknowledged", response.isAcknowledged()); ensureGreen("http_logs"); diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/WindowSqlIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/WindowSqlIT.java index 25660a2049d5d..1d37cea00b7bc 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/WindowSqlIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/sql/WindowSqlIT.java @@ -32,7 +32,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.test.OpenSearchIntegTestCase; @@ -67,7 +67,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfDerivedAboveProjectIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfDerivedAboveProjectIT.java index 0452d713f4529..4c234e015ece5 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfDerivedAboveProjectIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfDerivedAboveProjectIT.java @@ -24,11 +24,11 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.composite.CompositeDataFormatPlugin; import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; -import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; import java.time.LocalDate; import java.time.LocalDateTime; @@ -72,7 +72,7 @@ protected Collection additionalNodePlugins() { return List.of( classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), - classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) ); } @@ -199,7 +199,7 @@ private void createAndSeedIndex(int shardCount) { .indices() .prepareCreate(INDEX) .setSettings(indexSettings) - .setMapping("URL", "type=keyword", "EventDate", "type=date", "CounterID", "type=integer") + .setMapping("URL", "type=keyword,index=false", "EventDate", "type=date", "CounterID", "type=integer") .get(); assertTrue("index creation must be acknowledged", response.isAcknowledged()); ensureGreen(INDEX); diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java index 0eb362042a46b..3925f86dbbad2 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java @@ -104,8 +104,8 @@ public void testDumpQtfPipeline() throws Exception { Map> fields = new LinkedHashMap<>(); fields.put("CounterID", Map.of("type", "integer")); fields.put("UserID", Map.of("type", "long")); - fields.put("URL", Map.of("type", "keyword")); - fields.put("Title", Map.of("type", "keyword")); + fields.put("URL", Map.of("type", "keyword", "index", "false")); + fields.put("Title", Map.of("type", "keyword", "index", "false")); fields.put("EventDate", Map.of("type", "date")); ClusterState clusterState = clusterStateWith(INDEX, fields, "parquet", 2); @@ -192,8 +192,8 @@ private QueryDAG buildAndConvertQtfDag(String sql) { Map> fields = new LinkedHashMap<>(); fields.put("CounterID", Map.of("type", "integer")); fields.put("UserID", Map.of("type", "long")); - fields.put("URL", Map.of("type", "keyword")); - fields.put("Title", Map.of("type", "keyword")); + fields.put("URL", Map.of("type", "keyword", "index", "false")); + fields.put("Title", Map.of("type", "keyword", "index", "false")); fields.put("EventDate", Map.of("type", "date")); ClusterState clusterState = clusterStateWith(INDEX, fields, "parquet", 2); @@ -288,7 +288,7 @@ private static ClusterState clusterStateWith(String indexName, Map supportedFields() { + Set supported = new HashSet<>(super.supportedFields()); + Set metaSupported = Set.of( + new FieldTypeCapabilities(DocCountFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(RoutingFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IgnoredFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IdFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SeqNoFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, POINT_RANGE)), + new FieldTypeCapabilities(SeqNoFieldMapper.PRIMARY_TERM_NAME, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(VersionFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(IndexFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(NestedPathFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SourceFieldMapper.NAME, Set.of(STORED_FIELDS)), + new FieldTypeCapabilities(FieldNamesFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)) + ); + supported.removeIf(f -> containsMetaField(metaSupported, f.fieldType())); + supported.addAll(metaSupported); + return supported; + } + }; + } + + private boolean containsMetaField(Set fieldTypeCapabilities, String fieldType) { + for (FieldTypeCapabilities capability : fieldTypeCapabilities) { + if (capability.fieldType().equals(fieldType)) { + return true; + } + } + return false; + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AliasIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AliasIT.java index 016e2b4be7be9..6e8d628c9aacd 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AliasIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AliasIT.java @@ -215,6 +215,7 @@ private void createIndexIfAbsent(String name, String mappingJson) throws IOExcep "{\"settings\":{\"index.pluggable.dataformat.enabled\":true," + "\"index.pluggable.dataformat\":\"composite\"," + "\"index.composite.primary_data_format\":\"parquet\"," + + "\"index.composite.secondary_data_formats\":\"lucene\"," + "\"index.number_of_shards\":1,\"index.number_of_replicas\":0}," + "\"mappings\":" + mappingJson + "}" ); diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java index 22d8a41a432ae..40307bae8c501 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java @@ -142,7 +142,7 @@ public void testDistinctCountCrossShardOverlap() throws Exception { String body = "{\"settings\": {" + " \"number_of_shards\": " + shards + ", \"number_of_replicas\": 0," + " \"index.pluggable.dataformat.enabled\": true, \"index.pluggable.dataformat\": \"composite\"," - + " \"index.composite.primary_data_format\": \"parquet\", \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.primary_data_format\": \"parquet\", \"index.composite.secondary_data_formats\": \"lucene\"" + "}, \"mappings\": {\"properties\": {\"value\": {\"type\": \"integer\"}}}}"; Request create = new Request("PUT", "/" + index); create.setJsonEntity(body); @@ -193,7 +193,7 @@ public void testDistinctCountCrossShardOverlapKeyword() throws Exception { String body = "{\"settings\": {" + " \"number_of_shards\": " + shards + ", \"number_of_replicas\": 0," + " \"index.pluggable.dataformat.enabled\": true, \"index.pluggable.dataformat\": \"composite\"," - + " \"index.composite.primary_data_format\": \"parquet\", \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.primary_data_format\": \"parquet\", \"index.composite.secondary_data_formats\": \"lucene\"" + "}, \"mappings\": {\"properties\": {\"label\": {\"type\": \"keyword\"}}}}"; Request create = new Request("PUT", "/" + index); create.setJsonEntity(body); @@ -647,7 +647,7 @@ private void createStringGroupIndex() throws Exception { + " \"index.pluggable.dataformat.enabled\": true," + " \"index.pluggable.dataformat\": \"composite\"," + " \"index.composite.primary_data_format\": \"parquet\"," - + " \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.secondary_data_formats\": \"lucene\"" + "}," + "\"mappings\": {" + " \"properties\": {" @@ -720,7 +720,7 @@ private void createParquetBackedIndex(String indexName) throws Exception { + " \"index.pluggable.dataformat.enabled\": true," + " \"index.pluggable.dataformat\": \"composite\"," + " \"index.composite.primary_data_format\": \"parquet\"," - + " \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.secondary_data_formats\": \"lucene\"" + "}," + "\"mappings\": {" + " \"properties\": {" @@ -769,7 +769,7 @@ private void createSingleShardParquetBackedIndex(String indexName) throws Except + " \"index.pluggable.dataformat.enabled\": true," + " \"index.pluggable.dataformat\": \"composite\"," + " \"index.composite.primary_data_format\": \"parquet\"," - + " \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.secondary_data_formats\": \"lucene\"" + "}," + "\"mappings\": {" + " \"properties\": {" diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java index 724c377e2a0ff..55b31501ead99 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java @@ -66,7 +66,7 @@ private void createParquetBackedIndex() throws Exception { + " \"index.pluggable.dataformat.enabled\": true," + " \"index.pluggable.dataformat\": \"composite\"," + " \"index.composite.primary_data_format\": \"parquet\"," - + " \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.secondary_data_formats\": \"lucene\"" + "}," + "\"mappings\": {" + " \"properties\": {" diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DataStreamIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DataStreamIT.java index da2b20df29837..d50181222531e 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DataStreamIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DataStreamIT.java @@ -159,6 +159,7 @@ public void testCommaListMixingDataStreamAndConcreteIndex() throws IOException { "{\"settings\":{\"index.pluggable.dataformat.enabled\":true," + "\"index.pluggable.dataformat\":\"composite\"," + "\"index.composite.primary_data_format\":\"parquet\"," + + "\"index.composite.secondary_data_formats\":\"lucene\"," + "\"index.number_of_shards\":1,\"index.number_of_replicas\":0}," + "\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"v\":{\"type\":\"long\"}}}}" ); @@ -239,6 +240,7 @@ private void createDataStreamTemplate() throws IOException { + "\"index.pluggable.dataformat.enabled\":true," + "\"index.pluggable.dataformat\":\"composite\"," + "\"index.composite.primary_data_format\":\"parquet\"," + + "\"index.composite.secondary_data_formats\":\"lucene\"," + "\"index.number_of_shards\":1," + "\"index.number_of_replicas\":0" + "}," @@ -262,6 +264,7 @@ private void createDataStreamTemplateWithCategory() throws IOException { + "\"index.pluggable.dataformat.enabled\":true," + "\"index.pluggable.dataformat\":\"composite\"," + "\"index.composite.primary_data_format\":\"parquet\"," + + "\"index.composite.secondary_data_formats\":\"lucene\"," + "\"index.number_of_shards\":1," + "\"index.number_of_replicas\":0" + "}," diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldTypeCoverageIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldTypeCoverageIT.java index 867029d9e1c1f..c8f265a50f152 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldTypeCoverageIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldTypeCoverageIT.java @@ -167,7 +167,7 @@ public void testBinary() throws IOException { // BINARY(varchar) placeholder (BinaryFunctionAdapter rewrites it into a VARBINARY // literal that DataFusion compares natively). Filter coverage lives in testIpFilters // — binary columns share the same code path. - Map bulk = ingest("ft_binary", "binary", "\"YWxpY2U=\"", "\"Ym9i\"", "\"Y2Fyb2w=\""); + Map bulk = ingestWithMapping("ft_binary", "binary", ", \"store\": true", "\"YWxpY2U=\"", "\"Ym9i\"", "\"Y2Fyb2w=\""); assertBulkSucceeded(bulk, "ft_binary"); assertScanSucceeds("ft_binary", 3); } @@ -240,7 +240,7 @@ public void testIpAndBinaryProjectExpressions() throws IOException { ); assertFilterRowCount("source=ft_ip_project | stats count(eval(val='192.168.1.1')) as cnt", 1); - Map binBulk = ingest("ft_binary_project", "binary", "\"YWxpY2U=\"", "\"Ym9i\"", "\"Y2Fyb2w=\""); + Map binBulk = ingestWithMapping("ft_binary_project", "binary", ", \"store\": true", "\"YWxpY2U=\"", "\"Ym9i\"", "\"Y2Fyb2w=\""); assertBulkSucceeded(binBulk, "ft_binary_project"); assertFilterRowCount("source=ft_binary_project | eval is_alice=if(val='YWxpY2U=','y','n')", 3); assertFilterRowCount("source=ft_binary_project | stats count(eval(val='YWxpY2U=')) as c", 1); diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/IndexPatternUnionIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/IndexPatternUnionIT.java index 3d79cf9c4660c..aab90d0e6b6c3 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/IndexPatternUnionIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/IndexPatternUnionIT.java @@ -182,6 +182,7 @@ private void createParquetIndex(String name, String mappingJson) throws IOExcept "{\"settings\":{\"index.pluggable.dataformat.enabled\":true," + "\"index.pluggable.dataformat\":\"composite\"," + "\"index.composite.primary_data_format\":\"parquet\"," + + "\"index.composite.secondary_data_formats\":\"lucene\"," + "\"index.number_of_shards\":1,\"index.number_of_replicas\":0}," + "\"mappings\":" + mappingJson + "}" ); diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ListAggregateMultiTypeIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ListAggregateMultiTypeIT.java index 7f38c93ba6151..51c419a83ef2a 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ListAggregateMultiTypeIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ListAggregateMultiTypeIT.java @@ -140,7 +140,7 @@ private void provision() throws Exception { + " \"double_value\": { \"type\": \"double\" }," + " \"keyword_value\": { \"type\": \"keyword\" }," + " \"text_value\": { \"type\": \"text\" }," - + " \"binary_value\": { \"type\": \"binary\" }," + + " \"binary_value\": { \"type\": \"binary\", \"store\": \"true\" }," + " \"date_value\": { \"type\": \"date\", \"format\": \"yyyy-MM-dd HH:mm:ss\" }," + " \"date_nanos_value\": { \"type\": \"date_nanos\" }," + " \"ip_value\": { \"type\": \"ip\" }" diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultiIndexQueryShapesIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultiIndexQueryShapesIT.java index b34662bdce7aa..99c380800d501 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultiIndexQueryShapesIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultiIndexQueryShapesIT.java @@ -384,7 +384,7 @@ private void createIndexWithSettings(String name, String mappingJson, boolean wi } private void createIndexWithSettings(String name, String mappingJson, boolean withLuceneSecondary, int shards) throws IOException { - String secondaryFormats = withLuceneSecondary ? ",\"index.composite.secondary_data_formats\":\"lucene\"" : ""; + String secondaryFormats = ",\"index.composite.secondary_data_formats\":\"lucene\""; Request create = new Request("PUT", "/" + name); create.setJsonEntity( "{\"settings\":{\"index.pluggable.dataformat.enabled\":true," diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java index 5aa8bd05f6d18..0aa3bad9318e4 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java @@ -94,6 +94,7 @@ public void testFilterOnObjectField() throws IOException { } public void testFilterOnDeeplyNestedObjectField() throws IOException { + // This test treats latitude as a double, not geo point. assertRowsEqual( "source=" + DATASET.indexName + " | where city.location.latitude > 40 | fields city.name", row("Seattle"), diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java index 630fb18453193..566c04fc0dccd 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java @@ -47,7 +47,8 @@ public void testParquetIndexCreationAndIngestion() throws Exception { + " \"number_of_replicas\": 0," + " \"index.pluggable.dataformat.enabled\": true," + " \"index.pluggable.dataformat\": \"composite\"," - + " \"index.composite.primary_data_format\": \"parquet\"" + + " \"index.composite.primary_data_format\": \"parquet\"," + + " \"index.composite.secondary_data_formats\": \"lucene\"" + "}," + "\"mappings\": {" + " \"properties\": {" diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java index 2be85aca760fd..d3314d526a80b 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java @@ -261,7 +261,7 @@ private void createParquetBackedIndex() throws Exception { + " \"index.pluggable.dataformat.enabled\": true," + " \"index.pluggable.dataformat\": \"composite\"," + " \"index.composite.primary_data_format\": \"parquet\"," - + " \"index.composite.secondary_data_formats\": \"\"" + + " \"index.composite.secondary_data_formats\": \"lucene\"" + "}," + "\"mappings\": {" + " \"properties\": {" diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/extensive_coverage/mapping_test_data.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/extensive_coverage/mapping_test_data.json index 04cef9f43dda1..6c6cb86e3dfe4 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/extensive_coverage/mapping_test_data.json +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/extensive_coverage/mapping_test_data.json @@ -14,7 +14,6 @@ "tags": {"type": "keyword"}, "json_data": {"type": "text"}, "ip_address": {"type": "ip"}, - "location": {"type": "geo_point"}, "status": {"type": "keyword"}, "price": {"type": "double"}, "quantity": {"type": "integer"} diff --git a/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java index 2996306449554..fbbfc01ef6e07 100644 --- a/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java @@ -828,6 +828,10 @@ public void testDerivedSourceSimple() throws IOException { }, "ip_field": { "type": "ip" + }, + "binary_field": { + "type": "binary", + "store": true } } } @@ -849,6 +853,7 @@ public void testDerivedSourceSimple() throws IOException { .field("bool_field", true) .field("text_field", "test text") .field("ip_field", "1.2.3.4") + .field("binary_field", "bGlkaHQtd29rfx4=") .endObject() ) .get(); @@ -883,14 +888,16 @@ public void testDerivedSourceSimple() throws IOException { assertEquals(2, source.size()); assertEquals("test_keyword", source.get("keyword_field")); assertEquals(123, source.get("numeric_field")); + assertFalse(source.containsKey("binary_field")); // Test get with field exclusion getResponse = client().prepareGet("test_derive", "1").setFetchSource(null, new String[] { "text_field", "date_field" }).get(); assertTrue(getResponse.isExists()); source = getResponse.getSourceAsMap(); - assertEquals(5, source.size()); + assertEquals(6, source.size()); assertFalse(source.containsKey("text_field")); assertFalse(source.containsKey("date_field")); + assertTrue(source.containsKey("binary_field")); } public void testDerivedSource_MultiValuesAndComplexField() throws Exception { @@ -1088,6 +1095,7 @@ void validateDeriveSource(Map source) { assertEquals(true, source.get("bool_field")); assertEquals("test text", source.get("text_field")); assertEquals("1.2.3.4", source.get("ip_field")); + assertEquals("bGlkaHQtd29rfx4=", source.get("binary_field")); } void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolean sourceEnabled) { diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 3d69d1e7e6466..aaca83985e348 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -297,7 +297,8 @@ public IndexService( // we parse all percolator queries as they would be parsed on shard 0 () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, - scriptService + scriptService, + dataFormatRegistry ); this.indexFieldData = new IndexFieldDataService( indexSettings, diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index a7d01a08d56c0..08d717ec20ef4 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -1239,9 +1239,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti setMergeOnFlushPolicy(scopedSettings.get(INDEX_MERGE_ON_FLUSH_POLICY)); checkPendingFlushEnabled = scopedSettings.get(INDEX_CHECK_PENDING_FLUSH_ENABLED); defaultSearchPipeline = scopedSettings.get(DEFAULT_SEARCH_PIPELINE); - derivedSourceEnabled = scopedSettings.get(INDEX_DERIVED_SOURCE_SETTING); pluggableDataFormatEnabled = FeatureFlags.isEnabled(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) && scopedSettings.get(PLUGGABLE_DATAFORMAT_ENABLED_SETTING); + derivedSourceEnabled = scopedSettings.get(INDEX_DERIVED_SOURCE_SETTING) || pluggableDataFormatEnabled; pluggedDataFormat = scopedSettings.get(PLUGGABLE_DATAFORMAT_VALUE_SETTING); derivedSourceEnabledForTranslog = scopedSettings.get(INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer(INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING, this::setDerivedSourceEnabledForTranslog); diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java index 49821ac789e36..6b71e666217b9 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java @@ -11,8 +11,14 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.exec.commit.Committer; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperParsingException; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; /** @@ -64,6 +70,74 @@ default Map> getFormatDescriptors( return Map.of(); } + /** + * Assigns the capability map on the given field type. The plugin determines which capabilities + * its data format(s) can provide for the field type and sets the result via + * {@link MappedFieldType#setCapabilityMap}. + * + *

    The default implementation handles the single-format case: the plugin's own data format + * claims all capabilities it supports for the field type. Composite plugins override this to + * delegate to primary and secondary formats in priority order. + * + * @param fieldType the field type to assign capabilities to + * @param indexSettings the index settings + * @param dataFormatRegistry the registry, used by composite plugins to resolve sub-format plugins + * @throws MapperParsingException if the field type's requested capabilities cannot be fully covered + */ + default void assignCapabilities(MappedFieldType fieldType, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + Set requested = fieldType.requestedCapabilities(); + if (requested.isEmpty()) { + fieldType.setCapabilityMap(Map.of()); + return; + } + + final DataFormat format = getDataFormat(); + final String typeName = fieldType.typeName(); + final Set remaining = new HashSet<>(requested); + final Map> assigned = new LinkedHashMap<>(); + + final Set claimed = claimCapabilities(format, typeName, remaining); + if (claimed.isEmpty() == false) { + assigned.put(format, Set.copyOf(claimed)); + remaining.removeAll(claimed); + } + + if (remaining.isEmpty() == false) { + throw new MapperParsingException( + "Field [" + + fieldType.name() + + "] of type [" + + typeName + + "] requires capabilities " + + requested + + " but data format [" + + format.name() + + "] cannot cover: " + + remaining + ); + } + fieldType.setCapabilityMap(Map.copyOf(assigned)); + } + + /** + * Claims the capabilities that the given format supports for the specified field type. + */ + static Set claimCapabilities( + DataFormat format, + String typeName, + Set required + ) { + return format.supportedFields().stream().filter(ftc -> ftc.fieldType().equals(typeName)).findFirst().map(ftc -> { + Set intersection = EnumSet.noneOf(FieldTypeCapabilities.Capability.class); + for (FieldTypeCapabilities.Capability cap : required) { + if (ftc.capabilities().contains(cap)) { + intersection.add(cap); + } + } + return (Set) intersection; + }).orElse(Set.of()); + } + /** * Returns the strategies describing how this format participates in the tiered store, * keyed by the format name the strategy applies to. diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java index 6d51cfbc402ee..527bd3e6ffef2 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java @@ -15,6 +15,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.exec.EngineReaderManager; import org.opensearch.index.engine.exec.commit.Committer; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.plugins.PluginsService; import org.opensearch.plugins.SearchBackEndPlugin; @@ -182,6 +183,33 @@ public Map getStoreStrategies(IndexSettings indexSett return Map.of(); } + /** + * Assigns the capability map on the given field type by delegating to the configured data formats. + * Each format in priority order claims the capabilities it supports for the field type. + * If any requested capability remains unclaimed, a {@link org.opensearch.index.mapper.MapperParsingException} is thrown. + * + * @param fieldType the field type to assign capabilities to + * @param indexSettings the index settings used to resolve the active plugin + */ + public void assignCapabilities(MappedFieldType fieldType, IndexSettings indexSettings) { + String dataformatName = indexSettings.pluggableDataFormat(); + if (dataformatName == null || dataformatName.isEmpty()) { + fieldType.setCapabilityMap(Map.of()); + return; + } + DataFormat format = dataFormats.get(dataformatName); + if (format == null) { + fieldType.setCapabilityMap(Map.of()); + return; + } + DataFormatPlugin plugin = dataFormatPluginRegistry.get(format); + if (plugin == null) { + fieldType.setCapabilityMap(Map.of()); + return; + } + plugin.assignCapabilities(fieldType, indexSettings, this); + } + /** * Returns store strategies for a specific data format, bypassing the * {@code pluggable_dataformat} index setting lookup. Used by composite diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/FieldTypeCapabilities.java b/server/src/main/java/org/opensearch/index/engine/dataformat/FieldTypeCapabilities.java index 9408c4343a1ac..34a8ad62899b8 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/FieldTypeCapabilities.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/FieldTypeCapabilities.java @@ -45,6 +45,8 @@ public enum Capability { STORED_FIELDS, /** Probabilistic lookup for pruning*/ - BLOOM_FILTER + BLOOM_FILTER, + + FORWARD_TERMS_INDEX } } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/PrimaryTermFieldType.java b/server/src/main/java/org/opensearch/index/engine/exec/PrimaryTermFieldType.java index 80d74d84615aa..c6b2743c6a1c3 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/PrimaryTermFieldType.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/PrimaryTermFieldType.java @@ -37,7 +37,7 @@ public class PrimaryTermFieldType extends MappedFieldType { public static PrimaryTermFieldType INSTANCE = new PrimaryTermFieldType(); private PrimaryTermFieldType() { - super(SeqNoFieldMapper.PRIMARY_TERM_NAME, false, false, false, TextSearchInfo.NONE, Map.of()); + super(SeqNoFieldMapper.PRIMARY_TERM_NAME, false, false, true, TextSearchInfo.NONE, Map.of()); } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index 65c647693688b..10743e86e2173 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -101,13 +101,13 @@ public List> getParameters() { @Override public BinaryFieldMapper build(BuilderContext context) { - return new BinaryFieldMapper( - name, - new BinaryFieldType(buildFullName(context), stored.getValue(), hasDocValues.getValue(), meta.getValue()), - multiFieldsBuilder.build(this, context), - copyTo.build(), - this + final BinaryFieldType bft = new BinaryFieldType( + buildFullName(context), + stored.getValue(), + hasDocValues.getValue(), + meta.getValue() ); + return new BinaryFieldMapper(name, bft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -249,6 +249,21 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected void canDeriveSourceInternal() { + checkStoredForDerivedSource(); + } + + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, null, new StoredFieldFetcher(mappedFieldType, simpleName())) { + @Override + public FieldValueType getDerivedFieldPreference() { + return FieldValueType.STORED; + } + }; + } + /** * Custom binary doc values field for the binary field mapper * diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index 9ce7eaf751980..93c940ab4bbb5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -47,6 +47,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -342,6 +343,11 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } } + + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } } private final Boolean nullValue; diff --git a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java index 7664e855b0254..f1a698b5a7e61 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java @@ -27,6 +27,7 @@ import org.opensearch.common.regex.Regex; import org.opensearch.common.time.DateMathParser; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.ConstantIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -152,6 +153,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryShardCon return Regex.simpleMatch(pattern, value, caseInsensitive); } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } + @Override public Query existsQuery(QueryShardContext context) { return new MatchAllDocsQuery(); diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index dc769030adcdf..1448a362d92bf 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -60,6 +60,7 @@ import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.IndexSortConfig; import org.opensearch.index.compositeindex.datacube.DimensionType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -450,6 +451,11 @@ public String typeName() { return resolution.type(); } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.POINT_RANGE; + } + public DateFormatter dateTimeFormatter() { return dateTimeFormatter; } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java index 693f7e361d460..4c361f28408ed 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java @@ -39,6 +39,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchGenerationException; import org.opensearch.Version; +import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; @@ -51,6 +52,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.IndexSortConfig; import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.index.mapper.MetadataFieldMapper.TypeParser; @@ -64,6 +66,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import java.util.stream.Stream; /** @@ -93,8 +96,16 @@ public static class Builder { private final long newVersion; public Builder(RootObjectMapper.Builder builder, MapperService mapperService) { - final Settings indexSettings = mapperService.getIndexSettings().getSettings(); - this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1)); + this(builder, mapperService, null); + } + + public Builder(RootObjectMapper.Builder builder, MapperService mapperService, @Nullable DataFormatRegistry dataFormatRegistry) { + final IndexSettings is = mapperService.getIndexSettings(); + final Settings indexSettings = is.getSettings(); + final Consumer assigner = dataFormatRegistry != null + ? fieldType -> dataFormatRegistry.assignCapabilities(fieldType, is) + : null; + this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1), assigner); this.rootObjectMapper = builder.build(builderContext); final DocumentMapper existingMapper = mapperService.documentMapper(); @@ -116,6 +127,18 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) { } } + /** + * Recursively walks the mapper tree and assigns capability maps to all non-metadata field types. + */ + private static void assignCapabilitiesRecursive(Mapper mapper, Mapper.BuilderContext context) { + if (mapper instanceof FieldMapper) { + context.assignCapabilities(((FieldMapper) mapper).fieldType()); + } + for (Mapper child : mapper) { + assignCapabilitiesRecursive(child, context); + } + } + public Builder meta(Map meta) { this.meta = meta; return this; @@ -201,6 +224,16 @@ public DocumentMapper(MapperService mapperService, Mapping mapping, long version this.noopTombstoneMetadataFieldMappers = Stream.of(mapping.metadataMappers) .filter(field -> noopTombstoneMetadataFields.contains(field.name())) .toArray(MetadataFieldMapper[]::new); + + // Assign capabilities for dynamically merged mappers that bypass the Builder path. + final DataFormatRegistry registry = mapperService.documentMapperParser().getDataFormatRegistry(); + if (indexSettings.isPluggableDataFormatEnabled() && registry != null) { + assignCapabilitiesRecursive(mapping.root(), registry, indexSettings); + for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { + registry.assignCapabilities(metadataMapper.fieldType(), indexSettings); + } + } + this.version = version; } @@ -335,6 +368,26 @@ private boolean containSubDocIdWithObjectMapper(int nestedDocId, ObjectMapper ob } } + /** + * Recursively walks the mapper tree and assigns capability maps to all field types. + */ + private void assignCapabilitiesRecursive(Mapper mapper, DataFormatRegistry registry, IndexSettings indexSettings) { + if (mapper instanceof FieldMapper) { + registry.assignCapabilities(((FieldMapper) mapper).fieldType(), indexSettings); + // For derived source: keyword fields with ignore_above/normalizer use a separate + // rawValueFieldType to store the raw value for source reconstruction. + if (mapper instanceof KeywordFieldMapper keywordFieldMapper) { + KeywordFieldMapper.KeywordFieldType rawValueFieldType = keywordFieldMapper.getRawValueFieldType(); + if (rawValueFieldType != null && !mappers().isMultiField(keywordFieldMapper.fieldType().name())) { + registry.assignCapabilities(rawValueFieldType, indexSettings); + } + } + } + for (Mapper child : mapper) { + assignCapabilitiesRecursive(child, registry, indexSettings); + } + } + public DocumentMapper merge(Mapping mapping, MergeReason reason) { Mapping merged = this.mapping.merge(mapping, reason); return new DocumentMapper(mapperService, merged, this.version + 1L); diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java index 04c70e1205e13..c7bfd989d93cc 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java @@ -42,6 +42,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.mapper.MapperRegistry; @@ -74,6 +75,7 @@ public class DocumentMapperParser { private final Map typeParsers; private final Map rootTypeParsers; private final ScriptService scriptService; + private final DataFormatRegistry dataFormatRegistry; public DocumentMapperParser( IndexSettings indexSettings, @@ -83,6 +85,28 @@ public DocumentMapperParser( MapperRegistry mapperRegistry, Supplier queryShardContextSupplier, ScriptService scriptService + ) { + this( + indexSettings, + mapperService, + xContentRegistry, + similarityService, + mapperRegistry, + queryShardContextSupplier, + scriptService, + null + ); + } + + public DocumentMapperParser( + IndexSettings indexSettings, + MapperService mapperService, + NamedXContentRegistry xContentRegistry, + SimilarityService similarityService, + MapperRegistry mapperRegistry, + Supplier queryShardContextSupplier, + ScriptService scriptService, + @Nullable DataFormatRegistry dataFormatRegistry ) { this.mapperService = mapperService; this.xContentRegistry = xContentRegistry; @@ -92,6 +116,12 @@ public DocumentMapperParser( this.typeParsers = mapperRegistry.getMapperParsers(); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(); + this.dataFormatRegistry = dataFormatRegistry; + } + + @Nullable + DataFormatRegistry getDataFormatRegistry() { + return dataFormatRegistry; } public Mapper.TypeParser.ParserContext parserContext() { @@ -102,7 +132,8 @@ public Mapper.TypeParser.ParserContext parserContext() { indexVersionCreated, queryShardContextSupplier, null, - scriptService + scriptService, + Mapper.isPluggableDataFormatEnabled(mapperService.getIndexSettings().getSettings()) ? dataFormatRegistry : null ); } @@ -114,7 +145,8 @@ public Mapper.TypeParser.ParserContext parserContext(DateFormatter dateFormatter indexVersionCreated, queryShardContextSupplier, dateFormatter, - scriptService + scriptService, + Mapper.isPluggableDataFormatEnabled(mapperService.getIndexSettings().getSettings()) ? dataFormatRegistry : null ); } @@ -141,7 +173,8 @@ public DocumentMapper parse(String type, Map mapping) throws Map // parse RootObjectMapper DocumentMapper.Builder docBuilder = new DocumentMapper.Builder( (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), - mapperService + mapperService, + Mapper.isPluggableDataFormatEnabled(mapperService.getIndexSettings().getSettings()) ? dataFormatRegistry : null ); Iterator> iterator = mapping.entrySet().iterator(); // parse DocumentMapper diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldNamesFieldMapper.java index dc2ea1f35f8b8..46eae76995f9b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldNamesFieldMapper.java @@ -38,6 +38,7 @@ import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.lookup.SearchLookup; @@ -176,6 +177,11 @@ public Query termQuery(Object value, QueryShardContext context) { ); return super.termQuery(value, context); } + + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } } private final Explicit enabled; diff --git a/server/src/main/java/org/opensearch/index/mapper/FilterFieldType.java b/server/src/main/java/org/opensearch/index/mapper/FilterFieldType.java index 5029dd471813e..ecec066151f71 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FilterFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/FilterFieldType.java @@ -20,6 +20,8 @@ import org.opensearch.common.time.DateMathParser; import org.opensearch.common.unit.Fuzziness; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.query.IntervalMode; import org.opensearch.index.query.QueryRewriteContext; @@ -31,6 +33,7 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -290,4 +293,24 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S public MappedFieldType unwrap() { return delegate.unwrap(); } + + @Override + public Map> getCapabilityMap() { + return delegate.getCapabilityMap(); + } + + @Override + public synchronized void setCapabilityMap(Map> capabilityMap) { + delegate.setCapabilityMap(capabilityMap); + } + + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return delegate.searchCapability(); + } + + @Override + public Set requestedCapabilities() { + return delegate.requestedCapabilities(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java index dfc210cfd19f2..b97b332ae10f1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java @@ -194,18 +194,6 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(new BinaryDocValuesField(fieldType().name(), sketchBytes)); } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - byte[] value = parseHllValue(context); - if (value == null) { - return; - } - - BytesRef sketchBytes = new BytesRef(value); - validateSketchData(sketchBytes); - context.documentInput().addField(fieldType(), value); - } - private byte[] parseHllValue(ParseContext context) throws IOException { byte[] value = context.parseExternalValue(byte[].class); if (value == null) { diff --git a/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java index 9377b1f7dc6fe..74c122f90cc27 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java @@ -47,6 +47,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.util.BigArrays; import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.opensearch.index.fielddata.IndexFieldDataCache; @@ -146,6 +147,11 @@ public boolean isSearchable() { return true; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } + @Override public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); diff --git a/server/src/main/java/org/opensearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IgnoredFieldMapper.java index 0db795d99c7e0..8939de164da67 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IgnoredFieldMapper.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.lookup.SearchLookup; @@ -101,6 +102,11 @@ public Query existsQuery(QueryShardContext context) { return new TermRangeQuery(name(), null, null, true, true); } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } + @Override public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); diff --git a/server/src/main/java/org/opensearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IndexFieldMapper.java index 22fcb66848ed1..ecf459827d0f5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IndexFieldMapper.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.Query; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.Strings; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.ConstantIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -77,6 +78,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } + @Override protected boolean matches(String pattern, boolean caseInsensitive, QueryShardContext context) { if (caseInsensitive) { diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 144cee9f6034b..f51a2c8ea238c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -58,6 +58,7 @@ import org.opensearch.common.network.InetAddresses; import org.opensearch.common.network.NetworkAddress; import org.opensearch.index.compositeindex.datacube.DimensionType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; @@ -237,6 +238,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.POINT_RANGE; + } + private static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index a1ed5f8547a96..085f4a0ca00c0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -59,9 +59,11 @@ import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.compositeindex.datacube.DimensionType; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -174,14 +176,16 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter boost = Parameter.boostParam(); private final IndexAnalyzers indexAnalyzers; + private final boolean canConsumeRawValueForSource; - public Builder(String name, IndexAnalyzers indexAnalyzers) { + public Builder(String name, IndexAnalyzers indexAnalyzers, boolean canConsumeRawValueForSource) { super(name); this.indexAnalyzers = indexAnalyzers; + this.canConsumeRawValueForSource = canConsumeRawValueForSource; } public Builder(String name) { - this(name, null); + this(name, null, false); } public Builder ignoreAbove(int ignoreAbove) { @@ -267,11 +271,20 @@ public Optional getSupportedDataCubeDimensionType() { } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers())); + public static final TypeParser PARSER = new TypeParser( + (n, c) -> new Builder( + n, + c.getIndexAnalyzers(), + Optional.ofNullable(c.mapperService()) + .map(MapperService::getIndexSettings) + .map(IndexSettings::isPluggableDataFormatEnabled) + .orElse(false) + ) + ); @Override protected void canDeriveSourceInternal() { - if (this.ignoreAbove != Integer.MAX_VALUE || !Objects.equals(this.normalizerName, "default")) { + if (isIneligibleForGeneratingSource() && rawKeywordValueFieldType == null) { throw new UnsupportedOperationException( "Unable to derive source for [" + name() + "] with " + "ignore_above and/or normalizer set" ); @@ -279,6 +292,10 @@ protected void canDeriveSourceInternal() { checkStoredAndDocValuesForDerivedSource(); } + private boolean isIneligibleForGeneratingSource() { + return this.ignoreAbove != Integer.MAX_VALUE || !Objects.equals(this.normalizerName, "default"); + } + /** * 1. If it has doc values, build source using doc values * 2. If doc_values is disabled in field mapping, then build source using stored field @@ -392,6 +409,11 @@ NamedAnalyzer normalizer() { return indexAnalyzer(); } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); @@ -808,8 +830,10 @@ private void checkToDisableCaching(QueryShardContext context) { private final boolean useSimilarity; private final String normalizerName; private final boolean splitQueriesOnWhitespace; + private final KeywordFieldType rawKeywordValueFieldType; private final IndexAnalyzers indexAnalyzers; + private volatile boolean canConsumeRawValueForSource; protected KeywordFieldMapper( String simpleName, @@ -832,8 +856,10 @@ protected KeywordFieldMapper( this.useSimilarity = builder.useSimilarity.getValue(); this.normalizerName = builder.normalizer.getValue(); this.splitQueriesOnWhitespace = builder.splitQueriesOnWhitespace.getValue(); - this.indexAnalyzers = builder.indexAnalyzers; + this.canConsumeRawValueForSource = builder.canConsumeRawValueForSource; + this.rawKeywordValueFieldType = buildRawKeywordValueFieldType(); + } /** @@ -844,6 +870,31 @@ public int ignoreAbove() { return ignoreAbove; } + /** + * Returns the normalizer used for this keyword field. + * @return normalizerName + */ + public String normalizerName() { + return normalizerName; + } + + /** + * The field type to be used for derived source use cases. + * Keyword fields get ignored above a certain length and/or may get normalized. + * In such cases, storage layer would need to add another field which can be used for source generation + * @return sourceKeywordFieldType + */ + private KeywordFieldType buildRawKeywordValueFieldType() { + if (isIneligibleForGeneratingSource() && canConsumeRawValueForSource) { + return new KeywordFieldType("_ignored_source." + fieldType().name(), false, true, false, false, fieldType().meta()); + } + return null; + } + + public KeywordFieldType getRawValueFieldType() { + return rawKeywordValueFieldType; + } + boolean useSimilarity() { return useSimilarity; } @@ -883,14 +934,27 @@ protected void parseCreateField(ParseContext context) throws IOException { @Override protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - String value = parseKeywordValue(context); - if (value == null) { + String textValue = textValue(context); + if (textValue == null) { return; } - context.documentInput().addField(fieldType(), value); + String value = parseKeyword(textValue); + if (value != null) { + context.documentInput().addField(fieldType(), value); + } + // For derived source: store raw value separately when normalizer/ignore_above alters it. + // Skip for multi-field sub-fields (name contains dot) — parent field stores the raw source. + if (rawKeywordValueFieldType != null && (textValue.length() > ignoreAbove || !Objects.equals(normalizerName, "default"))) { + context.documentInput().addField(rawKeywordValueFieldType, textValue); + } } private String parseKeywordValue(ParseContext context) throws IOException { + String value = textValue(context); + return parseKeyword(value); + } + + private String textValue(ParseContext context) throws IOException { String value; if (context.externalValueSet()) { value = context.externalValue().toString(); @@ -902,7 +966,10 @@ private String parseKeywordValue(ParseContext context) throws IOException { value = parser.textOrNull(); } } + return value; + } + private String parseKeyword(String value) throws IOException { if (value == null || value.length() > ignoreAbove) { return null; } @@ -951,6 +1018,6 @@ protected String contentType() { @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexAnalyzers).init(this); + return new Builder(simpleName(), indexAnalyzers, canConsumeRawValueForSource).init(this); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java index a3ea6b5764913..6310a7f270fb5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java @@ -52,11 +52,14 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchParseException; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.time.DateMathParser; import org.opensearch.common.unit.Fuzziness; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.query.DistanceFeatureQueryBuilder; import org.opensearch.index.query.IntervalMode; @@ -69,9 +72,11 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -93,6 +98,13 @@ public abstract class MappedFieldType { private NamedAnalyzer indexAnalyzer; private boolean eagerGlobalOrdinals; + /** + * Capability map assigning each registered {@link DataFormat} to the set of capabilities it owns for this field type. + * Set once during mapping build via {@link Mapper.BuilderContext#assignCapabilities}, then immutable. + * Safe without volatile: publication through MapperService's volatile mapper reference provides happens-before. + */ + private Map> capabilityMap = Map.of(); + public MappedFieldType( String name, boolean isIndexed, @@ -466,6 +478,54 @@ public void setEagerGlobalOrdinals(boolean eagerGlobalOrdinals) { this.eagerGlobalOrdinals = eagerGlobalOrdinals; } + @ExperimentalApi + public Map> getCapabilityMap() { + return capabilityMap; + } + + /** + * Sets the capability map. + * Called by {@link org.opensearch.index.engine.dataformat.DataFormatRegistry} during mapping build. + * + * @throws IllegalStateException if already set + * @opensearch.experimental + */ + @ExperimentalApi + public synchronized void setCapabilityMap(Map> capabilityMap) { + this.capabilityMap = Map.copyOf(capabilityMap); + } + + /** + * Returns the search capability for this field type. Subclasses override to declare + * whether they use point-range (numerics, dates, IPs) or full-text (text, keyword) search. + * + * @opensearch.experimental + */ + @ExperimentalApi + protected FieldTypeCapabilities.Capability searchCapability() { + throw new UnsupportedOperationException("searchCapability is not supported for field: " + name() + " of type: " + typeName()); + } + + /** + * Derives the set of capabilities this field type requires based on its mapping parameters. + * + * @opensearch.experimental + */ + @ExperimentalApi + public Set requestedCapabilities() { + Set caps = new HashSet<>(); + if (isSearchable()) { + caps.add(searchCapability()); + } + if (hasDocValues()) { + caps.add(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE); + } + if (isStored()) { + caps.add(FieldTypeCapabilities.Capability.STORED_FIELDS); + } + return caps.isEmpty() ? Set.of() : Set.copyOf(caps); + } + /** Return a {@link DocValueFormat} that can be used to display and parse * values as returned by the fielddata API. * The default implementation returns a {@link DocValueFormat#RAW}. */ diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java index 939f7de12e912..cc19bb86d689d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java @@ -43,6 +43,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.similarity.SimilarityProvider; import org.opensearch.script.ScriptService; @@ -50,6 +51,7 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -72,11 +74,18 @@ public abstract class Mapper implements ToXContentFragment, Iterable { public static class BuilderContext { private final Settings indexSettings; private final ContentPath contentPath; + @Nullable + private final Consumer capabilityAssigner; public BuilderContext(Settings indexSettings, ContentPath contentPath) { + this(indexSettings, contentPath, (Consumer) null); + } + + public BuilderContext(Settings indexSettings, ContentPath contentPath, @Nullable Consumer capabilityAssigner) { Objects.requireNonNull(indexSettings, "indexSettings is required"); this.contentPath = contentPath; this.indexSettings = indexSettings; + this.capabilityAssigner = capabilityAssigner; } public ContentPath path() { @@ -87,6 +96,12 @@ public Settings indexSettings() { return this.indexSettings; } + public void assignCapabilities(MappedFieldType fieldType) { + if (capabilityAssigner != null) { + capabilityAssigner.accept(fieldType); + } + } + public Version indexCreatedVersion() { return IndexMetadata.indexCreated(indexSettings); } @@ -154,6 +169,8 @@ class ParserContext { private final ScriptService scriptService; + private final DataFormatRegistry dataFormatRegistry; + public ParserContext( Function similarityLookupService, MapperService mapperService, @@ -162,6 +179,28 @@ public ParserContext( Supplier queryShardContextSupplier, DateFormatter dateFormatter, ScriptService scriptService + ) { + this( + similarityLookupService, + mapperService, + typeParsers, + indexVersionCreated, + queryShardContextSupplier, + dateFormatter, + scriptService, + null + ); + } + + public ParserContext( + Function similarityLookupService, + MapperService mapperService, + Function typeParsers, + Version indexVersionCreated, + Supplier queryShardContextSupplier, + DateFormatter dateFormatter, + ScriptService scriptService, + @Nullable DataFormatRegistry dataFormatRegistry ) { this.similarityLookupService = similarityLookupService; this.mapperService = mapperService; @@ -170,12 +209,22 @@ public ParserContext( this.queryShardContextSupplier = queryShardContextSupplier; this.dateFormatter = dateFormatter; this.scriptService = scriptService; + this.dataFormatRegistry = dataFormatRegistry; } public IndexAnalyzers getIndexAnalyzers() { return mapperService.getIndexAnalyzers(); } + /** + * Returns the DataFormatRegistry, or null if not available. + * Only non-null when pluggable data format is enabled. + */ + @Nullable + public DataFormatRegistry dataFormatRegistry() { + return dataFormatRegistry; + } + public Settings getSettings() { return mapperService.getIndexSettings().getSettings(); } @@ -246,7 +295,8 @@ static class MultiFieldParserContext extends ParserContext { in.indexVersionCreated(), in.queryShardContextSupplier(), in.getDateFormatter(), - in.scriptService() + in.scriptService(), + in.dataFormatRegistry() ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index ccad43bd73c6e..788d725b7afd1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -38,6 +38,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.logging.DeprecationLogger; @@ -63,6 +64,7 @@ import org.opensearch.index.analysis.ReloadableCustomAnalyzer; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.mapper.Mapper.BuilderContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.similarity.SimilarityService; @@ -261,6 +263,30 @@ public MapperService( Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled, ScriptService scriptService + ) { + this( + indexSettings, + indexAnalyzers, + xContentRegistry, + similarityService, + mapperRegistry, + queryShardContextSupplier, + idFieldDataEnabled, + scriptService, + null + ); + } + + public MapperService( + IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + NamedXContentRegistry xContentRegistry, + SimilarityService similarityService, + MapperRegistry mapperRegistry, + Supplier queryShardContextSupplier, + BooleanSupplier idFieldDataEnabled, + ScriptService scriptService, + @Nullable DataFormatRegistry dataFormatRegistry ) { super(indexSettings); @@ -273,7 +299,8 @@ public MapperService( similarityService, mapperRegistry, queryShardContextSupplier, - scriptService + scriptService, + dataFormatRegistry ); this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); this.searchAnalyzer = new MapperAnalyzerWrapper( diff --git a/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java index 06db3fd38c062..c9a16aedd94c4 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.lookup.SearchLookup; @@ -106,5 +107,10 @@ public Query existsQuery(QueryShardContext context) { public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } + + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index ecf3e536896c8..5dff164b9532f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -63,6 +63,7 @@ import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.document.SortedUnsignedLongDocValuesRangeQuery; import org.opensearch.index.document.SortedUnsignedLongDocValuesSetQuery; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -1972,6 +1973,11 @@ public String typeName() { return type.name; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.POINT_RANGE; + } + public NumberType numberType() { return type; } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 25f895f0f358b..f1bbcf59190f0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -419,6 +419,9 @@ protected static void parseNested( if (type.equals(CONTENT_TYPE)) { builder.nested = Nested.NO; } else if (type.equals(NESTED_CONTENT_TYPE)) { + if (isPluggableDataFormatEnabled(parserContext.getSettings())) { + throw new MapperParsingException("nested type is not supported with pluggable data format on field [" + name + "]"); + } nested = true; } else { throw new MapperParsingException( diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index e02ba2b1bef6c..3bf6311d9cecf 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -51,6 +51,7 @@ import org.opensearch.common.time.DateMathParser; import org.opensearch.common.util.LocaleUtils; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.BinaryIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -345,6 +346,11 @@ public String typeName() { return rangeType.name; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.POINT_RANGE; + } + public DateFormatter dateTimeFormatter() { return dateTimeFormatter; } diff --git a/server/src/main/java/org/opensearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RoutingFieldMapper.java index 60decc56e0db2..4f1c88435883b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RoutingFieldMapper.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexOptions; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.lookup.SearchLookup; @@ -131,6 +132,11 @@ public String typeName() { public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } + + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } } private final boolean required; @@ -148,9 +154,14 @@ public boolean required() { public void preParse(ParseContext context) { String routing = context.sourceToParse().routing(); if (routing != null) { - context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE)); - createFieldNamesField(context); + if (context.indexSettings().isPluggableDataFormatEnabled()) { + context.documentInput().addField(this.fieldType(), routing); + } else { + context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE)); + createFieldNamesField(context); + } } + } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java index b99bd05345ce1..3e3222ff5df75 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java @@ -381,15 +381,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - String value = context.parser().textOrNull(); - if (value == null) { - return; - } - context.documentInput().addField(fieldType(), value); - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { Builder builder = new Builder(name()); diff --git a/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java index c7e9bed7577c5..756106bad76ff 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -130,6 +131,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.POINT_RANGE; + } + private long parse(Object value) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java index a01435db3a1e5..ccb93f031224b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java @@ -184,7 +184,9 @@ protected List> getParameters() { @Override public SourceFieldMapper build(BuilderContext context) { - if (context.indexSettings().getAsBoolean(IndexSettings.INDEX_DERIVED_SOURCE_SETTING.getKey(), false) && !enabled.getValue()) { + if ((context.indexSettings().getAsBoolean(IndexSettings.INDEX_DERIVED_SOURCE_SETTING.getKey(), false) + || context.indexSettings().getAsBoolean(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)) + && !enabled.getValue()) { throw new MapperParsingException( "_source can't be disabled with " + IndexSettings.INDEX_DERIVED_SOURCE_SETTING.getKey() + " enabled index setting" ); diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java index 1e7deb1dca4ed..12223cda20de3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java @@ -81,6 +81,7 @@ import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.opensearch.index.mapper.Mapper.TypeParser.ParserContext; @@ -472,7 +473,8 @@ protected PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); - if (context.indexSettings().getAsBoolean(IndexSettings.INDEX_DERIVED_SOURCE_SETTING.getKey(), false)) { + if (context.indexSettings().getAsBoolean(IndexSettings.INDEX_DERIVED_SOURCE_SETTING.getKey(), false) + || context.indexSettings().getAsBoolean(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)) { fieldType.setStored(true); } return new TextFieldMapper( @@ -838,6 +840,11 @@ void setIndexPhrases() { this.indexPhrases = true; } + @Override + protected FieldTypeCapabilities.Capability searchCapability() { + return FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; + } + public PrefixFieldType getPrefixFieldType() { return this.prefixFieldType; } diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginAssignCapabilitiesTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginAssignCapabilitiesTests.java new file mode 100644 index 0000000000000..f8ee5f9496233 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginAssignCapabilitiesTests.java @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.index.engine.dataformat.stub.MockDataFormat; +import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.Set; + +public class DataFormatPluginAssignCapabilitiesTests extends OpenSearchTestCase { + + public void testSingleFormatCoversAllCapabilities() { + MockDataFormat format = new MockDataFormat( + "keyword-format", + 100L, + Set.of( + new FieldTypeCapabilities( + "keyword", + Set.of(FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH, FieldTypeCapabilities.Capability.COLUMNAR_STORAGE) + ) + ) + ); + DataFormatPlugin plugin = MockDataFormatPlugin.of(format); + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("test_field"); + + plugin.assignCapabilities(fieldType, null, null); + + Map> capMap = fieldType.getCapabilityMap(); + assertEquals(1, capMap.size()); + assertTrue(capMap.containsKey(format)); + assertEquals( + Set.of(FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH, FieldTypeCapabilities.Capability.COLUMNAR_STORAGE), + capMap.get(format) + ); + } + + public void testSingleFormatPartialCoverage_Throws() { + MockDataFormat format = new MockDataFormat( + "partial-format", + 100L, + Set.of(new FieldTypeCapabilities("keyword", Set.of(FieldTypeCapabilities.Capability.COLUMNAR_STORAGE))) + ); + DataFormatPlugin plugin = MockDataFormatPlugin.of(format); + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("test_field"); + + MapperParsingException ex = expectThrows(MapperParsingException.class, () -> plugin.assignCapabilities(fieldType, null, null)); + assertTrue(ex.getMessage().contains("FULL_TEXT_SEARCH")); + } + + public void testFieldWithNoRequestedCapabilities_EmptyMap() { + MockDataFormat format = new MockDataFormat( + "some-format", + 100L, + Set.of(new FieldTypeCapabilities("integer", Set.of(FieldTypeCapabilities.Capability.POINT_RANGE))) + ); + DataFormatPlugin plugin = MockDataFormatPlugin.of(format); + NumberFieldMapper.NumberFieldType fieldType = new NumberFieldMapper.NumberFieldType( + "no_caps", + NumberFieldMapper.NumberType.INTEGER, + false, + false, + false, + false, + true, + null, + Map.of() + ); + + plugin.assignCapabilities(fieldType, null, null); + + assertTrue(fieldType.getCapabilityMap().isEmpty()); + } + + public void testFormatDoesNotSupportFieldType_Throws() { + MockDataFormat format = new MockDataFormat( + "integer-only-format", + 100L, + Set.of( + new FieldTypeCapabilities( + "integer", + Set.of(FieldTypeCapabilities.Capability.POINT_RANGE, FieldTypeCapabilities.Capability.COLUMNAR_STORAGE) + ) + ) + ); + DataFormatPlugin plugin = MockDataFormatPlugin.of(format); + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("test_field"); + + MapperParsingException ex = expectThrows(MapperParsingException.class, () -> plugin.assignCapabilities(fieldType, null, null)); + assertTrue(ex.getMessage().contains("cannot cover")); + } + + public void testFormatSupportsSubsetOnly_Throws() { + MockDataFormat format = new MockDataFormat( + "bloom-only-format", + 100L, + Set.of(new FieldTypeCapabilities("keyword", Set.of(FieldTypeCapabilities.Capability.BLOOM_FILTER))) + ); + DataFormatPlugin plugin = MockDataFormatPlugin.of(format); + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("test_field"); + + MapperParsingException ex = expectThrows(MapperParsingException.class, () -> plugin.assignCapabilities(fieldType, null, null)); + assertTrue(ex.getMessage().contains("FULL_TEXT_SEARCH")); + assertTrue(ex.getMessage().contains("COLUMNAR_STORAGE")); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java index 12b40a36947e3..9f62eda8c2fdb 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java @@ -145,7 +145,7 @@ public void testPluggableDataFormatBinaryValue() throws Exception { Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); DocumentMapper mapper = createDocumentMapper( pluggableSettings, - mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()) + mapping(b -> b.startObject("field").field("type", "binary").field("store", true).endObject()) ); CapturingDocumentInput docInput = new CapturingDocumentInput(); byte[] testValue = new byte[] { 1, 2, 3 }; @@ -161,7 +161,7 @@ public void testPluggableDataFormatBinaryNullSkipped() throws Exception { Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); DocumentMapper mapper = createDocumentMapper( pluggableSettings, - mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()) + mapping(b -> b.startObject("field").field("type", "binary").field("store", true).endObject()) ); CapturingDocumentInput docInput = new CapturingDocumentInput(); mapper.parse(source(b -> b.nullField("field")), docInput); @@ -179,7 +179,7 @@ public void testPluggablePathEquivalenceWithLucenePath() throws Exception { String base64Value = java.util.Base64.getEncoder().encodeToString(testValue); assertBinaryLuceneAndPluggablePathsEquivalent( pluggableSettings, - mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()), + mapping(b -> b.startObject("field").field("type", "binary").field("store", true).endObject()), b -> b.field("field", base64Value), "field", true @@ -188,7 +188,7 @@ public void testPluggablePathEquivalenceWithLucenePath() throws Exception { // Scenario 2: null value — no field produced assertBinaryLuceneAndPluggablePathsEquivalent( pluggableSettings, - mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()), + mapping(b -> b.startObject("field").field("type", "binary").field("store", true).endObject()), b -> b.nullField("field"), "field", false diff --git a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java index 99a75d58496cb..f854ce5d3cc51 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java @@ -46,9 +46,7 @@ import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; -import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; @@ -792,16 +790,4 @@ protected V featureValueOf(T actual) { } }; } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatCompletionNoOp() throws IOException { - Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "suggestion")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("CompletionFieldMapper pluggable format is no-op, should not capture", hasField); - } - } diff --git a/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java index 8eb9c6a526702..9dab7d1162ce3 100644 --- a/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java @@ -10,10 +10,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitMixer; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.metrics.AbstractHyperLogLog; @@ -22,8 +20,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.List; -import java.util.Map; import static org.opensearch.search.DocValueFormat.BINARY; import static org.hamcrest.Matchers.containsString; @@ -769,61 +765,4 @@ public void testValueFetcherWithDocValues() throws IOException { // 2. Sketches can be reconstructed from doc values // 3. DocValueFormat is BINARY } - - private Settings pluggableSettings() { - return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); - } - - private byte[] createValidSketchBytes() throws IOException { - HyperLogLogPlusPlus sketch = new HyperLogLogPlusPlus(HyperLogLogPlusPlus.DEFAULT_PRECISION, BigArrays.NON_RECYCLING_INSTANCE, 1); - try { - sketch.collect(0, 1L); - BytesStreamOutput out = new BytesStreamOutput(); - sketch.writeTo(0, out); - BytesRef ref = out.bytes().toBytesRef(); - return Arrays.copyOfRange(ref.bytes, ref.offset, ref.offset + ref.length); - } finally { - sketch.close(); - } - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatValidSketch() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "hll").endObject()) - ); - byte[] sketchBytes = createValidSketchBytes(); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", sketchBytes)), docInput); - - List> captured = docInput.getCapturedFields(); - boolean found = captured.stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Expected hll field captured with sketch bytes", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatNullValueSkipped() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "hll").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatInvalidSketchThrows() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "hll").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - byte[] invalidBytes = new byte[] { 0, 1, 2, 3 }; - Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", invalidBytes)), docInput)); - assertThat(e.getMessage(), containsString("field [field]")); - } } diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java index 2123869841fb9..585b7dc2ba1d9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java @@ -475,7 +475,7 @@ public void testFetchSourceValue() throws IOException { assertEquals(Collections.singletonList("42"), fetchSourceValue(ignoreAboveMapper, 42L)); assertEquals(Collections.singletonList("true"), fetchSourceValue(ignoreAboveMapper, true)); - MappedFieldType normalizerMapper = new KeywordFieldMapper.Builder("field", createIndexAnalyzers()).normalizer("lowercase") + MappedFieldType normalizerMapper = new KeywordFieldMapper.Builder("field", createIndexAnalyzers(), false).normalizer("lowercase") .build(context) .fieldType(); assertEquals(Collections.singletonList("value"), fetchSourceValue(normalizerMapper, "VALUE")); diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java index e4a2806f3fed5..65264dcf3df08 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java @@ -46,9 +46,7 @@ import java.io.IOException; import java.net.InetAddress; -import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import static org.opensearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -461,45 +459,4 @@ public void testUpdatesWithSameMappings() throws Exception { private Settings pluggableSettings() { return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatLongRange() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "long_range").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.startObject("field").field("gte", 5).field("lte", 10).endObject()), docInput); - - List> captured = docInput.getCapturedFields(); - boolean found = captured.stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Expected range field captured", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatNullValueSkipped() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "long_range").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatIpRange() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "ip_range").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "192.168.1.0/24")), docInput); - - List> captured = docInput.getCapturedFields(); - boolean found = captured.stream().anyMatch(e -> e.getKey().name().equals("field")); - assertTrue("Expected ip_range field captured from CIDR", found); - } } diff --git a/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java index 5864d5c181fa8..bdfcc68b2f41a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java @@ -31,7 +31,6 @@ import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -857,32 +856,4 @@ public void testFieldTypeErrorCases() { private Settings pluggableSettings() { return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatSemanticVersion() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "version").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "1.2.3")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("1.2.3")); - assertTrue("Expected version field captured with value '1.2.3'", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatNullValueSkipped() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "version").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } } diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java index 29da0f00ac2a9..cbab0cf6fb0b9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java @@ -28,9 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; @@ -402,175 +400,4 @@ private Document createDocument(WildcardFieldMapper mapper, String value) throws private Settings pluggableSettings() { return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatDefaultWildcard() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "wildcard").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "test_value")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("test_value")); - assertTrue("Expected wildcard field captured with value 'test_value'", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatNullValueSkipped() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "wildcard").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field for null value", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatNullValueConfigured() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "wildcard").field("null_value", "default_val").endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.nullField("field")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("default_val")); - assertTrue("Expected wildcard field captured with null_value 'default_val'", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatIgnoreAbove() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 5).endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "opensearch")), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("Expected no captured field when value exceeds ignore_above", hasField); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatIgnoreAboveWithinLimit() throws IOException { - DocumentMapper mapper = createDocumentMapper( - pluggableSettings(), - mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 10).endObject()) - ); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", "elk")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("elk")); - assertTrue("Expected wildcard field captured with value 'elk' within ignore_above limit", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatWithExternalValue() throws IOException { - DocumentMapper mapper = createDocumentMapper(pluggableSettings(), mapping(b -> { - b.startObject("text_field"); - b.field("type", "text"); - b.startObject("fields"); - b.startObject("wc").field("type", "wildcard").endObject(); - b.endObject(); - b.endObject(); - })); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("text_field", "external_wildcard")), docInput); - - boolean found = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals("text_field.wc") && e.getValue().equals("external_wildcard")); - assertTrue("Expected wildcard sub-field captured with external value", found); - } - - @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggablePathEquivalenceWithLucenePath() throws IOException { - Settings pluggable = pluggableSettings(); - - // Scenario 1: default wildcard value - assertWildcardLuceneAndPluggablePathsEquivalent( - pluggable, - mapping(b -> b.startObject("field").field("type", "wildcard").endObject()), - b -> b.field("field", "test_value"), - "field", - "test_value" - ); - - // Scenario 2: null value — no field produced - assertWildcardLuceneAndPluggablePathsEquivalent( - pluggable, - mapping(b -> b.startObject("field").field("type", "wildcard").endObject()), - b -> b.nullField("field"), - "field", - null - ); - - // Scenario 3: null_value configured — substitution kicks in - assertWildcardLuceneAndPluggablePathsEquivalent( - pluggable, - mapping(b -> b.startObject("field").field("type", "wildcard").field("null_value", "default_val").endObject()), - b -> b.nullField("field"), - "field", - "default_val" - ); - - // Scenario 4: ignore_above — value exceeds limit, no field produced - assertWildcardLuceneAndPluggablePathsEquivalent( - pluggable, - mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 5).endObject()), - b -> b.field("field", "opensearch"), - "field", - null - ); - - // Scenario 5: ignore_above — value within limit - assertWildcardLuceneAndPluggablePathsEquivalent( - pluggable, - mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 10).endObject()), - b -> b.field("field", "elk"), - "field", - "elk" - ); - } - - private void assertWildcardLuceneAndPluggablePathsEquivalent( - Settings pluggableSettings, - XContentBuilder mappingBuilder, - CheckedConsumer sourceBuilder, - String fieldName, - String expectedValue - ) throws IOException { - // Lucene path - DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); - ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); - IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); - - // Pluggable path - DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); - CapturingDocumentInput docInput = new CapturingDocumentInput(); - pluggableMapper.parse(source(sourceBuilder), docInput); - - if (expectedValue == null) { - assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); - boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); - assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); - } else { - assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); - - boolean pluggableFound = docInput.getCapturedFields() - .stream() - .anyMatch(e -> e.getKey().name().equals(fieldName) && e.getValue().equals(expectedValue)); - assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); - } - } } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/AbstractDataFormatAwareEngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/AbstractDataFormatAwareEngineTestCase.java index 4cc0b92add29c..2d2fe5c71dcef 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/AbstractDataFormatAwareEngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/AbstractDataFormatAwareEngineTestCase.java @@ -57,6 +57,8 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -361,7 +363,8 @@ public void testFlushCommitsAndPreservesData() throws IOException { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); + Engine.IndexResult result = engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); + assertNull("expected no failure but got: " + result.getFailure(), result.getFailure()); } engine.flush(false, true); @@ -476,6 +479,7 @@ public void testConcurrentIndexThenRefreshAndFlush() throws Exception { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int totalDocs = randomIntBetween(20, 50); AtomicInteger failures = new AtomicInteger(0); + List exceptions = Collections.synchronizedList(new ArrayList<>()); // Index concurrently int numThreads = randomIntBetween(2, 4); @@ -487,18 +491,25 @@ public void testConcurrentIndexThenRefreshAndFlush() throws Exception { try { barrier.await(); int idx; + Exception failure; while ((idx = docCounter.getAndIncrement()) < totalDocs) { - engine.index(indexOp(createParsedDocWithInput(Integer.toString(idx), null))); + if ((failure = engine.index(indexOp(createParsedDocWithInput(Integer.toString(idx), null))) + .getFailure()) != null) { + throw failure; + } } } catch (Exception e) { failures.incrementAndGet(); + exceptions.add(e); } }); threads[t].start(); } for (Thread t : threads) t.join(); - assertThat(failures.get(), equalTo(0)); + if (failures.get() > 0) { + fail("concurrent indexing failed with " + failures.get() + " exceptions" + exceptions); + } // Refresh and flush engine.refresh("after-indexing"); diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/DataFormatTestUtils.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/DataFormatTestUtils.java new file mode 100644 index 0000000000000..81a9b425589aa --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/DataFormatTestUtils.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.index.mapper.MappedFieldType; + +import java.util.Map; + +/** + * Test utilities for the pluggable data format capability system. + */ +public final class DataFormatTestUtils { + + private DataFormatTestUtils() {} + + /** + * Assigns the capability map on a {@link MappedFieldType} so that the given {@link DataFormat} + * owns the capabilities declared in its {@link DataFormat#supportedFields()} for the field's type name. + * Simulates what {@code FieldCapabilityAssigner.assign} does at mapping build time. + */ + public static void assignTestCapabilities(MappedFieldType fieldType, DataFormat format) { + format.supportedFields() + .stream() + .filter(ftc -> ftc.fieldType().equals(fieldType.typeName())) + .findFirst() + .ifPresent(ftc -> fieldType.setCapabilityMap(Map.of(format, ftc.capabilities()))); + } +} diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/FileBackedDataFormatPlugin.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/FileBackedDataFormatPlugin.java index 0c9a0011378e2..a869fa1add249 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/FileBackedDataFormatPlugin.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/FileBackedDataFormatPlugin.java @@ -31,7 +31,19 @@ import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.commit.IndexStoreProvider; +import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.FieldNamesFieldMapper; +import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.IgnoredFieldMapper; +import org.opensearch.index.mapper.IndexFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NestedPathFieldMapper; +import org.opensearch.index.mapper.RoutingFieldMapper; +import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; +import org.opensearch.index.mapper.TextFieldMapper; +import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.index.store.PrecomputedChecksumStrategy; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchBackEndPlugin; @@ -48,6 +60,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.POINT_RANGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.STORED_FIELDS; + /** * A file-backed data format plugin for integration tests. Format name: "filebacked". * Writers persist doc IDs as lines in text files. Supports failure injection via static methods. @@ -69,7 +86,21 @@ public long priority() { @Override public Set supportedFields() { - return Set.of(); + return Set.of( + new FieldTypeCapabilities(KeywordFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(TextFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(DocCountFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(RoutingFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IgnoredFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IdFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SeqNoFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, POINT_RANGE)), + new FieldTypeCapabilities(SeqNoFieldMapper.PRIMARY_TERM_NAME, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(VersionFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(IndexFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(NestedPathFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SourceFieldMapper.NAME, Set.of(STORED_FIELDS)), + new FieldTypeCapabilities(FieldNamesFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)) + ); } }; diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java index 8404e6e022149..58238564d6caa 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java @@ -8,14 +8,53 @@ package org.opensearch.index.engine.dataformat.stub; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.FieldNamesFieldMapper; +import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.IgnoredFieldMapper; +import org.opensearch.index.mapper.IndexFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; +import org.opensearch.index.mapper.RoutingFieldMapper; +import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.index.mapper.SourceFieldMapper; +import org.opensearch.index.mapper.TextFieldMapper; +import org.opensearch.index.mapper.VersionFieldMapper; + import java.util.Set; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.COLUMNAR_STORAGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.FULL_TEXT_SEARCH; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.POINT_RANGE; +import static org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability.STORED_FIELDS; + /** * A mock {@link MockDataFormatPlugin} that registers "parquet" as a data format for testing. */ public class MockParquetDataFormatPlugin extends MockDataFormatPlugin { public MockParquetDataFormatPlugin() { - super(new MockDataFormat("parquet", 100L, Set.of())); + super( + new MockDataFormat( + "parquet", + 100L, + Set.of( + new FieldTypeCapabilities(KeywordFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(TextFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(DocCountFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(RoutingFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IgnoredFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(IdFieldMapper.CONTENT_TYPE, Set.of(STORED_FIELDS, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SeqNoFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, POINT_RANGE)), + new FieldTypeCapabilities(SeqNoFieldMapper.PRIMARY_TERM_NAME, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(VersionFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE)), + new FieldTypeCapabilities(IndexFieldMapper.CONTENT_TYPE, Set.of(COLUMNAR_STORAGE, FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(NestedPathFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)), + new FieldTypeCapabilities(SourceFieldMapper.NAME, Set.of(STORED_FIELDS)), + new FieldTypeCapabilities(FieldNamesFieldMapper.NAME, Set.of(FULL_TEXT_SEARCH)) + ) + ) + ); } } From 58d612df638657b9b2a804f06095be00f156b008 Mon Sep 17 00:00:00 2001 From: Finn Date: Sat, 6 Jun 2026 11:51:08 -0700 Subject: [PATCH 02/11] Log DataFusion execution metrics on data nodes at DEBUG level (#21999) After fragment execution completes on a data node, extract DataFusion's per-operator execution metrics (output_rows, elapsed_compute, time_elapsed_scanning_total, row_groups_pruned, bytes_scanned, etc.) and log them at DEBUG level. This provides visibility into DataFusion execution performance without requiring the full profile API transport changes. Implementation: - Rust: new FFM function df_stream_get_metrics extracts ExecutionPlan.metrics() as JSON bytes. QueryStreamHandle stores the physical plan Arc for post-execution metrics access. - Java: NativeBridge.streamGetMetrics() calls the FFM function. DatafusionResultStream implements FragmentResources.MetricsCapable. FragmentResources.getExecutionMetrics() returns the JSON bytes. - AnalyticsSearchService: after stream iterator is exhausted, calls getExecutionMetrics() and logs at DEBUG level. To see metrics, set logger level: PUT /_cluster/settings {"transient":{"logger.org.opensearch.analytics.exec.AnalyticsSearchService":"DEBUG"}} Signed-off-by: Finn Carroll --- .../rust/src/api.rs | 49 +++++++++++++++++++ .../rust/src/ffm.rs | 31 ++++++++++++ .../rust/src/indexed_executor.rs | 4 +- .../rust/src/query_executor.rs | 13 +++-- .../be/datafusion/DatafusionResultStream.java | 8 ++- .../be/datafusion/nativelib/NativeBridge.java | 36 ++++++++++++++ .../exec/AnalyticsSearchService.java | 11 +++++ .../analytics/exec/FragmentResources.java | 17 +++++++ 8 files changed, 161 insertions(+), 8 deletions(-) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index a59f486cae6b9..0bf126101aa6c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -83,6 +83,9 @@ pub struct QueryStreamHandle { /// Concurrency gate permit — held for the query's entire lifetime. /// Released on drop, which frees partition budget for other queries. _concurrency_permit: Option, + /// Physical plan reference for post-execution metrics extraction. + /// Available after execution completes; read via `df_stream_get_metrics`. + physical_plan: Option>, } impl QueryStreamHandle { @@ -104,6 +107,7 @@ impl QueryStreamHandle { _session_ctx: None, has_views, _concurrency_permit: permit, + physical_plan: None, } } @@ -120,6 +124,51 @@ impl QueryStreamHandle { _session_ctx: Some(ctx), has_views, _concurrency_permit: permit, + physical_plan: None, + } + } + + pub fn with_physical_plan( + stream: RecordBatchStreamAdapter, + query_context: QueryTrackingContext, + ctx: datafusion::prelude::SessionContext, + permit: Option, + plan: Arc, + ) -> Self { + let has_views = Self::schema_has_views(&stream.schema()); + Self { + stream, + _query_tracking_context: query_context, + _session_ctx: Some(ctx), + has_views, + _concurrency_permit: permit, + physical_plan: Some(plan), + } + } + + /// Returns execution metrics from ALL operators in the physical plan tree as JSON bytes. + /// Walks the tree recursively, collecting metrics from every node. + pub fn get_metrics_json(&self) -> Option> { + let plan = self.physical_plan.as_ref()?; + let mut map = serde_json::Map::new(); + Self::collect_metrics(plan.as_ref(), &mut map); + if map.is_empty() { + return None; + } + serde_json::to_vec(&map).ok() + } + + fn collect_metrics(plan: &dyn datafusion::physical_plan::ExecutionPlan, map: &mut serde_json::Map) { + if let Some(metrics) = plan.metrics() { + for m in metrics.iter() { + let name = m.value().name().to_string(); + let value = m.value().as_usize() as i64; + // Later operators override earlier ones if same name — leaf (scan) metrics take priority + map.insert(name, serde_json::Value::Number(serde_json::Number::from(value))); + } + } + for child in plan.children() { + Self::collect_metrics(child.as_ref(), map); } } } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 8ab6c94826623..fc201d1397176 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -354,6 +354,37 @@ pub unsafe extern "C" fn df_stream_close(stream_ptr: i64) { api::stream_close(stream_ptr); } +/// Returns execution metrics as JSON bytes for the given stream. +/// Writes the pointer to allocated bytes into `out_ptr` and the length into `out_len_ptr`. +/// Returns 0 on success, non-zero if no metrics are available. +/// The caller must free the returned bytes via `df_free_metrics_buf`. +#[no_mangle] +pub unsafe extern "C" fn df_stream_get_metrics(stream_ptr: i64, out_ptr: *mut *const u8, out_len_ptr: *mut i64) -> i64 { + if stream_ptr == 0 { + return -1; + } + let handle = &*(stream_ptr as *const api::QueryStreamHandle); + match handle.get_metrics_json() { + Some(bytes) => { + let len = bytes.len() as i64; + let boxed = bytes.into_boxed_slice(); + let ptr = Box::into_raw(boxed) as *const u8; + *out_ptr = ptr; + *out_len_ptr = len; + 0 + } + None => -1, + } +} + +/// Frees a metrics buffer previously returned by `df_stream_get_metrics`. +#[no_mangle] +pub unsafe extern "C" fn df_free_metrics_buf(ptr: *mut u8, len: i64) { + if !ptr.is_null() && len > 0 { + let _ = Box::from_raw(std::slice::from_raw_parts_mut(ptr, len as usize)); + } +} + #[no_mangle] pub extern "C" fn df_cancel_query(context_id: i64) { api::cancel_query(context_id); diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs index 8c96eb106be0a..d368305f625bf 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs @@ -886,7 +886,7 @@ async unsafe fn execute_indexed_with_context_inner( let target_schema = crate::schema_coerce::coerce_inferred_schema(physical_plan.schema()); let physical_plan = crate::relabel_exec::wrap_if_relabel_needed(physical_plan, target_schema)?; log_debug!("DataFusion physical plan:\n{}", displayable(physical_plan.as_ref()).indent(true)); - let df_stream = execute_stream(physical_plan, ctx.task_ctx()) + let df_stream = execute_stream(physical_plan.clone(), ctx.task_ctx()) .map_err(|e| DataFusionError::Execution(format!("execute_stream: {}", e)))?; let (cross_rt_stream, abort_handle) = @@ -898,6 +898,6 @@ async unsafe fn execute_indexed_with_context_inner( let schema = cross_rt_stream.schema(); let wrapped = RecordBatchStreamAdapter::new(schema, cross_rt_stream); - let stream_handle = crate::api::QueryStreamHandle::with_session_context(wrapped, query_context, ctx, Some(permit)); + let stream_handle = crate::api::QueryStreamHandle::with_physical_plan(wrapped, query_context, ctx, Some(permit), physical_plan); Ok(Box::into_raw(Box::new(stream_handle)) as i64) } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index 22a74bc660364..e7c9272fb4683 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -328,7 +328,7 @@ pub async fn execute_with_context( cross_rt_stream.schema(), cross_rt_stream, ); - return Ok::(Box::into_raw(Box::new(wrapped)) as i64); + return Ok::<(i64, Option>), DataFusionError>((Box::into_raw(Box::new(wrapped)) as i64, None)); } let dataframe = handle.ctx.execute_logical_plan(logical_plan).await?; @@ -339,7 +339,7 @@ pub async fn execute_with_context( let target_schema = crate::schema_coerce::coerce_inferred_schema(physical_plan.schema()); let physical_plan = crate::relabel_exec::wrap_if_relabel_needed(physical_plan, target_schema)?; - let df_stream = execute_stream(physical_plan, handle.ctx.task_ctx()).map_err(|e| { + let df_stream = execute_stream(physical_plan.clone(), handle.ctx.task_ctx()).map_err(|e| { error!("execute_with_context: failed to create stream: {}", e); e })?; @@ -356,10 +356,10 @@ pub async fn execute_with_context( cross_rt_stream, ); - Ok::(Box::into_raw(Box::new(wrapped)) as i64) + Ok::<(i64, Option>), DataFusionError>((Box::into_raw(Box::new(wrapped)) as i64, Some(physical_plan))) }; - let stream_ptr = crate::cancellation::cancellable(token.as_ref(), context_id, query_future) + let (stream_ptr, physical_plan) = crate::cancellation::cancellable(token.as_ref(), context_id, query_future) .await .map_err(|e| DataFusionError::Execution(e))?; @@ -367,7 +367,10 @@ pub async fn execute_with_context( let stream = unsafe { *Box::from_raw(stream_ptr as *mut datafusion::physical_plan::stream::RecordBatchStreamAdapter) }; // Permit is held until the QueryStreamHandle is dropped (query complete). // If cancellation fires → stream drops → handle drops → permit drops → gate releases. - let stream_handle = crate::api::QueryStreamHandle::with_session_context(stream, handle.query_context, handle.ctx, Some(permit)); + let stream_handle = match physical_plan { + Some(plan) => crate::api::QueryStreamHandle::with_physical_plan(stream, handle.query_context, handle.ctx, Some(permit), plan), + None => crate::api::QueryStreamHandle::with_session_context(stream, handle.query_context, handle.ctx, Some(permit)), + }; Ok(Box::into_raw(Box::new(stream_handle)) as i64) } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java index bfd61175e66bc..3c04ec48bf037 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java @@ -21,6 +21,7 @@ import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.backend.EngineResultStream; import org.opensearch.analytics.exec.ArrowValues; +import org.opensearch.analytics.exec.FragmentResources; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.StreamHandle; import org.opensearch.common.annotation.ExperimentalApi; @@ -42,7 +43,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class DatafusionResultStream implements EngineResultStream { +public class DatafusionResultStream implements EngineResultStream, FragmentResources.MetricsCapable { private final StreamHandle streamHandle; private final BufferAllocator allocator; @@ -64,6 +65,11 @@ public Iterator iterator() { return iteratorInstance; } + @Override + public byte[] getMetricsJson() { + return NativeBridge.streamGetMetrics(streamHandle.getPointer()); + } + @Override public void close() { try { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index c7401e59f99d2..5d7604edbf1a1 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -85,6 +85,8 @@ public final class NativeBridge { private static final MethodHandle STREAM_GET_SCHEMA; private static final MethodHandle STREAM_NEXT; private static final MethodHandle STREAM_CLOSE; + private static final MethodHandle STREAM_GET_METRICS; + private static final MethodHandle FREE_METRICS_BUF; private static final MethodHandle SQL_TO_SUBSTRAIT; private static final MethodHandle REGISTER_FILTER_TREE_CALLBACKS; private static final MethodHandle CREATE_LOCAL_SESSION; @@ -240,6 +242,15 @@ public final class NativeBridge { STREAM_CLOSE = linker.downcallHandle(lib.find("df_stream_close").orElseThrow(), FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG)); + STREAM_GET_METRICS = linker.downcallHandle( + lib.find("df_stream_get_metrics").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.ADDRESS) + ); + + FREE_METRICS_BUF = linker.downcallHandle( + lib.find("df_free_metrics_buf").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); // i64 df_sql_to_substrait(shard_ptr, table_ptr, table_len, sql_ptr, sql_len, runtime_ptr, out_ptr, out_cap, out_len) SQL_TO_SUBSTRAIT = linker.downcallHandle( lib.find("df_sql_to_substrait").orElseThrow(), @@ -867,6 +878,31 @@ public static void streamClose(long streamPtr) { NativeCall.invokeVoid(STREAM_CLOSE, streamPtr); } + /** + * Extracts execution metrics from the stream's physical plan as JSON bytes. + * Returns null if no metrics are available (e.g., plan didn't capture them). + * Must be called BEFORE streamClose (the stream handle must still be alive). + */ + public static byte[] streamGetMetrics(long streamPtr) { + try (var arena = Arena.ofConfined()) { + var outPtr = arena.allocate(ValueLayout.ADDRESS); + var outLen = arena.allocate(ValueLayout.JAVA_LONG); + long result = (long) STREAM_GET_METRICS.invokeExact(streamPtr, outPtr, outLen); + if (result != 0) { + return null; // no metrics available + } + long len = outLen.get(ValueLayout.JAVA_LONG, 0); + MemorySegment dataPtr = outPtr.get(ValueLayout.ADDRESS, 0); + byte[] bytes = dataPtr.reinterpret(len).toArray(ValueLayout.JAVA_BYTE); + // Free the Rust-allocated buffer + FREE_METRICS_BUF.invokeExact(dataPtr, len); + return bytes; + } catch (Throwable t) { + logger.debug("Failed to read native stream metrics", t); + return null; + } + } + // ---- Cancellation ---- /** Fires the cancellation token for the given context. No-op if already completed. */ diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java index 43ed1dc0f1983..bb7fd7501a2ce 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java @@ -176,6 +176,17 @@ public void executeFragmentStreamingAsync( responseHandler.onBatch(batch); } long fragmentTookNanos = System.nanoTime() - startNanos; + // Extract and log DataFusion execution metrics at DEBUG level + if (LOGGER.isDebugEnabled()) { + byte[] metricsJson = exec.resources().getExecutionMetrics(); + if (metricsJson != null) { + LOGGER.debug( + "[FragmentMetrics] shard={} metrics={}", + shard.shardId(), + new String(metricsJson, java.nio.charset.StandardCharsets.UTF_8) + ); + } + } responseHandler.onComplete(); ResolvedFragment resolved = exec.resolved(); DelegationDescriptor delegation = resolved.plan().getDelegationDescriptor(); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java index 7ddfa371bea20..4d9a2bb0d2209 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java @@ -75,6 +75,23 @@ public EngineResultStream stream() { return stream; } + /** + * Extracts execution metrics from the underlying engine stream (if supported). + * Must be called after the stream is exhausted but before close(). + * Returns null if the stream doesn't support metrics extraction. + */ + public byte[] getExecutionMetrics() { + if (stream instanceof MetricsCapable mc) { + return mc.getMetricsJson(); + } + return null; + } + + /** Marker interface for streams that can provide execution metrics. */ + public interface MetricsCapable { + byte[] getMetricsJson(); + } + @Override public void close() throws Exception { // Close the stream and engine first so any in-flight release upcalls from native From b0d39c812c4b5281c59133426136acfcd11a9493 Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Sat, 6 Jun 2026 14:03:20 -0700 Subject: [PATCH 03/11] feat: read oversampling factor from node opensearch.yml (#22023) Signed-off-by: Sandesh Kumar --- .../analytics/exec/DefaultPlanExecutor.java | 16 +++- .../analytics/planner/PlannerContext.java | 9 +++ .../planner/rules/OpenSearchTopKRewriter.java | 7 +- .../planner/TopKRewriterPlanShapeTests.java | 6 +- sandbox/qa/analytics-engine-rest/build.gradle | 19 +++++ .../analytics/qa/YmlOversamplingIT.java | 77 +++++++++++++++++++ 6 files changed, 120 insertions(+), 14 deletions(-) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index 2b05b47a779d9..48c721381160f 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -39,6 +39,7 @@ import org.opensearch.analytics.planner.dag.PlanAlternativeSelector; import org.opensearch.analytics.planner.dag.PlanForker; import org.opensearch.analytics.planner.dag.QueryDAG; +import org.opensearch.analytics.settings.AnalyticsApproximationSettings; import org.opensearch.analytics.settings.AnalyticsQuerySettings; import org.opensearch.analytics.stats.AnalyticsStatsCollector; import org.opensearch.arrow.allocator.AllocationRejection; @@ -95,6 +96,7 @@ public class DefaultPlanExecutor extends HandledTransportAction preferMetadataDriver = v); + this.oversamplingFactor = AnalyticsApproximationSettings.SHARD_BUCKET_OVERSAMPLING_FACTOR.get(clusterService.getSettings()); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(AnalyticsApproximationSettings.SHARD_BUCKET_OVERSAMPLING_FACTOR, v -> oversamplingFactor = v); this.indexNameExpressionResolver = indexNameExpressionResolver; this.analyticsSearchSlowLog = analyticsSearchSlowLog; } @@ -227,10 +232,15 @@ private void executeInternal( // TODO: remove the null fallback once every front-end (test-ppl-frontend, // dsl-query-executor) threads an EngineContextProvider.getContext() snapshot through. ClusterState planningState = queryCtx != null ? queryCtx.clusterState() : clusterService.state(); - RelNode plan = PlannerImpl.createPlan( - logicalFragment, - new PlannerContext(capabilityRegistry, planningState, indexNameExpressionResolver, false, preferMetadataDriver) + PlannerContext plannerContext = new PlannerContext( + capabilityRegistry, + planningState, + indexNameExpressionResolver, + false, + preferMetadataDriver ); + plannerContext.setOversamplingFactor(oversamplingFactor); + RelNode plan = PlannerImpl.createPlan(logicalFragment, plannerContext); final String fullPlan = profile ? org.apache.calcite.plan.RelOptUtil.toString(plan) : null; QueryDAG dag = DAGBuilder.build(plan, capabilityRegistry, clusterService, indexNameExpressionResolver); PlanForker.forkAll(dag, capabilityRegistry); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerContext.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerContext.java index 46633086d04e7..4544ff630f11a 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerContext.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerContext.java @@ -29,6 +29,7 @@ public class PlannerContext { private final OpenSearchDistributionTraitDef distributionTraitDef; private final boolean profilingEnabled; private final boolean preferMetadataDriver; + private double oversamplingFactor; private int annotationIdCounter; private RuleProfilingListener.PlannerProfile lastProfile; @@ -103,6 +104,14 @@ public ClusterState getClusterState() { return clusterState; } + public double getOversamplingFactor() { + return oversamplingFactor; + } + + public void setOversamplingFactor(double oversamplingFactor) { + this.oversamplingFactor = oversamplingFactor; + } + public OpenSearchDistributionTraitDef getDistributionTraitDef() { return distributionTraitDef; } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java index 93755457be72f..a2f580f3e6334 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java @@ -23,9 +23,7 @@ import org.opensearch.analytics.planner.rel.OpenSearchExchangeReducer; import org.opensearch.analytics.planner.rel.OpenSearchProject; import org.opensearch.analytics.planner.rel.OpenSearchSort; -import org.opensearch.analytics.settings.AnalyticsApproximationSettings; import org.opensearch.analytics.spi.AggregateFunction; -import org.opensearch.common.settings.Settings; import java.util.ArrayList; import java.util.List; @@ -211,10 +209,7 @@ private static RelNode replaceInTree(RelNode root, RelNode oldNode, RelNode newN } private static double resolveOversamplingFactor(PlannerContext context) { - // TODO: Move to per-index setting once index-pattern/alias resolution is handled. - Settings clusterSettings = context.getClusterState().metadata().settings(); - if (clusterSettings == null) return 0.0; - return AnalyticsApproximationSettings.SHARD_BUCKET_OVERSAMPLING_FACTOR.get(clusterSettings); + return context.getOversamplingFactor(); } private record SortAboveFinal(OpenSearchSort sort, OpenSearchAggregate finalAgg) { diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java index 11c0f61417580..c114d678c8996 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java @@ -19,12 +19,9 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; -import org.opensearch.common.settings.Settings; import java.util.List; -import static org.mockito.Mockito.when; - /** * Plan shape tests for the TopK rewriter. Grouped into Detection (skip cases) * and Rewrite (insertion cases). @@ -255,8 +252,7 @@ private RelNode buildBareLimitOverGroupedCount() { private PlannerContext contextWithOversampling(double factor) { PlannerContext ctx = buildContext("parquet", 2, intFields()); - Settings clusterSettings = Settings.builder().put("analytics.shard_bucket_oversampling_factor", factor).build(); - when(ctx.getClusterState().metadata().settings()).thenReturn(clusterSettings); + ctx.setOversamplingFactor(factor); return ctx; } } diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle index 2e1623c806f3c..3dad4a724bd8e 100644 --- a/sandbox/qa/analytics-engine-rest/build.gradle +++ b/sandbox/qa/analytics-engine-rest/build.gradle @@ -121,6 +121,7 @@ integTest { exclude '**/CoordinatorReduceMemtableIT.class' exclude '**/StreamingCoordinatorReduceIT.class' exclude '**/QueryCacheIT.class' + exclude '**/YmlOversamplingIT.class' // Note: parallel forks against the same 2-node testCluster slow the suite down // (cluster is the bottleneck, not JVM startup) and cause cross-fork data races @@ -181,6 +182,24 @@ testClusters.integTestStreaming { configureAnalyticsCluster(delegate) } +// ── YML oversampling variant: verifies TopK fires from opensearch.yml setting ─ +task integTestYmlOversampling(type: RestIntegTestTask) { + description = 'Verifies TopK oversampling works when set via opensearch.yml (node settings)' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + filter { + includeTestsMatching 'org.opensearch.analytics.qa.YmlOversamplingIT' + } + systemProperty 'tests.security.manager', 'false' +} +check.dependsOn(integTestYmlOversampling) + +testClusters.integTestYmlOversampling { + numberOfNodes = 2 + configureAnalyticsCluster(delegate) + setting 'analytics.shard_bucket_oversampling_factor', '2.0' +} + // Run against an external cluster (no testClusters lifecycle): // ./gradlew :sandbox:qa:analytics-engine-rest:restTest // ./gradlew :sandbox:qa:analytics-engine-rest:restTest -PrestCluster=host:port diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java new file mode 100644 index 0000000000000..d4a131a6bd773 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import java.util.List; +import java.util.Map; + +/** + * Verifies that {@code analytics.shard_bucket_oversampling_factor} set via opensearch.yml + * (node settings at bootstrap) is picked up by the TopK rewriter without any dynamic + * cluster settings API call. + * + *

    The test cluster for this IT is configured in build.gradle with: + * {@code setting 'analytics.shard_bucket_oversampling_factor', '2.0'} + */ +public class YmlOversamplingIT extends AnalyticsRestTestCase { + + private static final String INDEX = "yml_oversampling_test"; + + /** + * Grouped count + sort + head without any runtime cluster settings call. + * If TopK fires (oversampling from yml), the query succeeds with correct results. + * If TopK doesn't fire, it still succeeds but without per-shard pruning. + * We verify correctness: exact count values on a known dataset. + */ + public void testTopKFiresFromYmlSetting() throws Exception { + // Create 2-shard index + createParquetIndex(INDEX, 2); + + // 3 groups: A=6, B=4, C=2 docs + StringBuilder bulk = new StringBuilder(); + for (int i = 0; i < 6; i++) bulk.append("{\"index\":{}}\n{\"grp\":\"A\"}\n"); + for (int i = 0; i < 4; i++) bulk.append("{\"index\":{}}\n{\"grp\":\"B\"}\n"); + for (int i = 0; i < 2; i++) bulk.append("{\"index\":{}}\n{\"grp\":\"C\"}\n"); + bulkAndRefresh(INDEX, bulk.toString()); + + // No cluster settings API call — oversampling should come from yml + Map result = executePpl( + "source = " + INDEX + " | stats count() as c by grp | sort - c | head 2" + ); + @SuppressWarnings("unchecked") + List> rows = (List>) result.get("datarows"); + assertEquals("top-2 groups", 2, rows.size()); + assertEquals("top group count", 6, ((Number) rows.get(0).get(0)).intValue()); + assertEquals("top group key", "A", rows.get(0).get(1)); + assertEquals("second group count", 4, ((Number) rows.get(1).get(0)).intValue()); + assertEquals("second group key", "B", rows.get(1).get(1)); + } + + private void createParquetIndex(String index, int shards) throws Exception { + try { client().performRequest(new org.opensearch.client.Request("DELETE", "/" + index)); } catch (Exception ignored) {} + String body = "{\"settings\": {" + + " \"number_of_shards\": " + shards + ", \"number_of_replicas\": 0," + + " \"index.pluggable.dataformat.enabled\": true, \"index.pluggable.dataformat\": \"composite\"," + + " \"index.composite.primary_data_format\": \"parquet\", \"index.composite.secondary_data_formats\": \"\"" + + "}, \"mappings\": {\"properties\": {\"grp\": {\"type\": \"keyword\"}}}}"; + org.opensearch.client.Request create = new org.opensearch.client.Request("PUT", "/" + index); + create.setJsonEntity(body); + assertOkAndParse(client().performRequest(create), "create " + index); + org.opensearch.client.Request health = new org.opensearch.client.Request("GET", "/_cluster/health/" + index); + health.addParameter("wait_for_status", "green"); + health.addParameter("timeout", "30s"); + client().performRequest(health); + } + + private void bulkAndRefresh(String index, String body) throws Exception { + org.opensearch.client.Request bulk = new org.opensearch.client.Request("POST", "/" + index + "/_bulk?refresh=true"); + bulk.setJsonEntity(body); + client().performRequest(bulk); + } +} From ac95dd40d1e08b229972f2ca7665eae6479b4e42 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Sat, 6 Jun 2026 17:51:39 -0700 Subject: [PATCH 04/11] =?UTF-8?q?fix(analytics-engine):=20zero=20null=20te?= =?UTF-8?q?xt=20cells=20before=20passing=20results=20to=E2=80=A6=20(#22027?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(analytics-engine): zero null text cells before passing results to native A QTF query could crash randomly when results had null text values. The null cells held leftover garbage memory, and the native engine read that garbage as if it were a real string and crashed. Fix: clear the null cells before handing results to the native engine. Signed-off-by: Marc Handalian * Move null view sanitization into the Stitcher (QTF producer) Run it on the hand-built QTF output instead of in the shared reduce sinks, so only the small K-row batch is scanned and the generic path is untouched. Signed-off-by: Marc Handalian * Fix Rust build: prepared-plan path must return the (ptr, plan) tuple The upstream tuple refactor of execute_with_context updated the empty-shard and normal exits to return (i64, Option>) but missed the prepared_plan early-return, which still returned a bare i64 — so the closure's return type didn't match the tuple destructure and failed to compile (E0308). The prepared path has no physical_plan to surface, so it returns None. Signed-off-by: Marc Handalian * Move null view sanitization into VectorUtils Fold ViewVectorSanitizer into VectorUtils.sanitizeNullViewSlots alongside the other VectorSchemaRoot helpers; the Stitcher calls it before feeding its output. Signed-off-by: Marc Handalian * Mute YmlOversamplingIT (fails 100%) @AwaitsFix testTopKFiresFromYmlSetting: the opensearch.yml-bootstrap oversampling factor is not picked up by the TopK rewriter. Needs investigation. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../rust/src/query_executor.rs | 6 +- .../analytics/exec/VectorUtils.java | 32 +++++ .../analytics/exec/stage/Stitcher.java | 2 + .../analytics/exec/VectorUtilsTests.java | 118 ++++++++++++++++++ .../analytics/qa/YmlOversamplingIT.java | 3 + 5 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/VectorUtilsTests.java diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index e7c9272fb4683..e695c7d2bed23 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -293,7 +293,11 @@ pub async fn execute_with_context( cross_rt_stream.schema(), cross_rt_stream, ); - return Ok::(Box::into_raw(Box::new(wrapped)) as i64); + // Prepared (engine-native PARTIAL) path carries no physical_plan handle — match the + // tuple shape of the other exits so the closure's return type stays consistent. + return Ok::<(i64, Option>), DataFusionError>( + (Box::into_raw(Box::new(wrapped)) as i64, None), + ); } let substrait_plan = Plan::decode(plan_bytes).map_err(|e| { diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/VectorUtils.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/VectorUtils.java index b4d91bed8b989..36fc046def79a 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/VectorUtils.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/VectorUtils.java @@ -8,7 +8,9 @@ package org.opensearch.analytics.exec; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -75,4 +77,34 @@ public static VectorSchemaRoot appendConstantInt(VectorSchemaRoot input, String combined.add(constantVector); return new VectorSchemaRoot(combined); } + + /** + * Zeroes the view of every null slot in a hand-built batch's view columns (Utf8View / + * BinaryView), making the batch safe to export across the Arrow C Data Interface. + * + *

    When a producer fills a view vector via {@code copyFromSafe}/{@code setNull}, Arrow unsets + * the validity bit but leaves the slot's 16-byte view as uninitialized buffer memory. Arrow-Java + * honors the validity bit, but a raw consumer reading the layout (DataFusion's interleave) reads + * the view length prefix unconditionally — a garbage length > 12 dereferences a variadic data + * buffer that does not exist on an all-null column, panicking. Zeroing makes each null slot a + * valid length-0 inline view; validity bits are untouched, so cells stay null. Mutates in place, + * so a downstream zero-copy export still hands the same buffers to native. + * + *

    Only needed for batches built in Java (e.g. the QTF stitch output); batches imported from + * native are already spec-valid. + * + * @param root batch to sanitize in place; not closed by this method + */ + public static void sanitizeNullViewSlots(VectorSchemaRoot root) { + int rows = root.getRowCount(); + for (FieldVector vec : root.getFieldVectors()) { + if (!(vec instanceof BaseVariableWidthViewVector view) || view.getNullCount() == 0) continue; + ArrowBuf viewBuffer = view.getDataBuffer(); + for (int row = 0; row < rows; row++) { + if (view.isNull(row)) { + viewBuffer.setZero((long) row * BaseVariableWidthViewVector.ELEMENT_SIZE, BaseVariableWidthViewVector.ELEMENT_SIZE); + } + } + } + } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/Stitcher.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/Stitcher.java index 9a191c7b06e3e..c7991bc1ca3f4 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/Stitcher.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/Stitcher.java @@ -15,6 +15,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.exec.VectorUtils; import org.opensearch.analytics.planner.rel.OpenSearchLateMaterialization; import org.opensearch.analytics.spi.ExchangeSink; @@ -171,6 +172,7 @@ private void finish() { try { if (failures.isEmpty()) { output.setRowCount(totalRows); + VectorUtils.sanitizeNullViewSlots(output); parentSink.feed(output); parentSink.close(); logger.debug("[Stitcher] emitted rows={}", totalRows); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/VectorUtilsTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/VectorUtilsTests.java new file mode 100644 index 0000000000000..56c3722788787 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/VectorUtilsTests.java @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.exec; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BaseVariableWidthViewVector; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * Unit tests for {@link VectorUtils#sanitizeNullViewSlots}. + * + *

    Regression for the QTF late-materialization {@code interleave len=0, index=} panic. + * A Java-built VSR fed across the Arrow C Data Interface can carry view columns (Utf8View / + * BinaryView) whose null slots were never written — Arrow unsets the validity bit but leaves the + * 16-byte view as uninitialized garbage. DataFusion's interleave reads the view length prefix + * unconditionally; a garbage length > 12 dereferences a variadic data buffer that does not exist + * on an all-null column, panicking. The sanitizer must zero every null view slot before export so + * it reads back as a valid length-0 inline view. + */ +public class VectorUtilsTests extends OpenSearchTestCase { + + private static final int ELEMENT_SIZE = BaseVariableWidthViewVector.ELEMENT_SIZE; + private static final long GARBAGE_VIEW_LENGTH = 0x7FFFFFFFL; // far above INLINE_SIZE (12) + + private BufferAllocator allocator; + + @Override + public void setUp() throws Exception { + super.setUp(); + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @Override + public void tearDown() throws Exception { + allocator.close(); + super.tearDown(); + } + + /** A null Utf8View slot left with a garbage length prefix must be zeroed; the cell stays null. */ + public void testSanitize_zeroesGarbageNullUtf8ViewSlot() { + try (ViewVarCharVector v = new ViewVarCharVector("v", allocator)) { + v.allocateNew(64, 3); + v.setSafe(0, "hi".getBytes(StandardCharsets.UTF_8)); + v.getDataBuffer().setInt((long) 1 * ELEMENT_SIZE, (int) GARBAGE_VIEW_LENGTH); + v.setNull(1); + v.getDataBuffer().setInt((long) 2 * ELEMENT_SIZE, (int) GARBAGE_VIEW_LENGTH); + v.setNull(2); + v.setValueCount(3); + assertEquals(GARBAGE_VIEW_LENGTH, Integer.toUnsignedLong(v.getDataBuffer().getInt((long) 1 * ELEMENT_SIZE))); + + try (VectorSchemaRoot root = new VectorSchemaRoot(List.of(v))) { + VectorUtils.sanitizeNullViewSlots(root); + + assertEquals(0, v.getDataBuffer().getInt((long) 1 * ELEMENT_SIZE)); + assertEquals(0, v.getDataBuffer().getInt((long) 2 * ELEMENT_SIZE)); + assertTrue(v.isNull(1)); + assertTrue(v.isNull(2)); + assertFalse(v.isNull(0)); + assertEquals("hi", new String(v.get(0), StandardCharsets.UTF_8)); + } + } + } + + /** BinaryView shares the same 16-byte layout and the same hazard — also covered. */ + public void testSanitize_zeroesGarbageNullBinaryViewSlot() { + try (ViewVarBinaryVector v = new ViewVarBinaryVector("b", allocator)) { + v.allocateNew(64, 2); + v.setSafe(0, new byte[] { 1, 2, 3 }); + v.getDataBuffer().setInt((long) 1 * ELEMENT_SIZE, (int) GARBAGE_VIEW_LENGTH); + v.setNull(1); + v.setValueCount(2); + + try (VectorSchemaRoot root = new VectorSchemaRoot(List.of(v))) { + VectorUtils.sanitizeNullViewSlots(root); + + assertEquals(0, v.getDataBuffer().getInt((long) 1 * ELEMENT_SIZE)); + assertTrue(v.isNull(1)); + assertFalse(v.isNull(0)); + } + } + } + + /** Non-view columns (and view columns with no nulls) are left untouched. */ + public void testSanitize_ignoresNonViewAndNonNullColumns() { + try (BigIntVector ints = new BigIntVector("i", allocator); ViewVarCharVector v = new ViewVarCharVector("v", allocator)) { + ints.allocateNew(2); + ints.setSafe(0, 10); + ints.setSafe(1, 20); + ints.setValueCount(2); + v.allocateNew(64, 2); + v.setSafe(0, "a".getBytes(StandardCharsets.UTF_8)); + v.setSafe(1, "bb".getBytes(StandardCharsets.UTF_8)); + v.setValueCount(2); + + try (VectorSchemaRoot root = new VectorSchemaRoot(List.of(ints, v))) { + VectorUtils.sanitizeNullViewSlots(root); // no nulls anywhere — must be a no-op, no exception + + assertEquals(10, ints.get(0)); + assertEquals("a", new String(v.get(0), StandardCharsets.UTF_8)); + assertEquals("bb", new String(v.get(1), StandardCharsets.UTF_8)); + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java index d4a131a6bd773..3d088c3b72304 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/YmlOversamplingIT.java @@ -8,6 +8,8 @@ package org.opensearch.analytics.qa; +import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; + import java.util.List; import java.util.Map; @@ -29,6 +31,7 @@ public class YmlOversamplingIT extends AnalyticsRestTestCase { * If TopK doesn't fire, it still succeeds but without per-shard pruning. * We verify correctness: exact count values on a known dataset. */ + @AwaitsFix(bugUrl = "Fails 100% — yml-bootstrap oversampling factor not picked up by the TopK rewriter; needs investigation") public void testTopKFiresFromYmlSetting() throws Exception { // Create 2-shard index createParquetIndex(INDEX, 2); From 5be7579bfb4a821b775106f0e646e7d451371756 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Sat, 6 Jun 2026 20:03:47 -0700 Subject: [PATCH 05/11] Push bare LIMIT down to shards (head N without ORDER BY) (#22029) A head N with no sort left the limit only on the coordinator, so every shard streamed its full scan and the coordinator trimmed to N. Push a collation-free fetch below the exchange so each shard caps at N locally. Safe without an order: the coordinator's N rows are a subset of the union of each shard's N rows. Extends the existing collated-sort pushdown to the bare-fetch case, reusing its offset handling. Plan-shape tests cover the pushed shape plus single-shard, offset-widening, and UNION ALL. Updates testPureLimit_2shard, which had pinned the old shape. Signed-off-by: Marc Handalian --- .../rules/OpenSearchSortPushdownRewriter.java | 5 ++ .../analytics/planner/SortPlanShapeTests.java | 5 +- .../planner/SortPushdownPlanShapeTests.java | 73 +++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchSortPushdownRewriter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchSortPushdownRewriter.java index c5e39e4d23b8f..5e3d72e805783 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchSortPushdownRewriter.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchSortPushdownRewriter.java @@ -73,6 +73,11 @@ private static RewriteContext find(RelNode node, OpenSearchSort fetchSortAbove) if (bound != null && canComputeShardFetch(bound) && isPushTarget(below)) { return new RewriteContext(sort, bound, below); } + } else if (sort.fetch != null && canComputeShardFetch(sort) && isPushTarget(sort.getInput())) { + // Bare LIMIT, no ORDER BY (`head N`): push a fetch-only Sort below the ER so each + // shard caps at N instead of streaming its whole scan. Safe with no order — the + // coordinator just needs some N rows, and each shard's local N supplies them. + return new RewriteContext(sort, sort, sort.getInput()); } OpenSearchSort carry = (collated == false && sort.fetch != null) ? sort : null; return find(sort.getInput(), carry); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPlanShapeTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPlanShapeTests.java index b87d9078b87fe..f49a38652e580 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPlanShapeTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPlanShapeTests.java @@ -72,6 +72,8 @@ public void testPureLimit_1shard() { """, result); } + /** A bare LIMIT is pushed below the ER so each shard caps locally instead of streaming its whole + * scan to the coordinator. See {@link SortPushdownPlanShapeTests#testBareLimit_2shard_pushed}. */ public void testPureLimit_2shard() { RelNode scan = stubScan(mockTable("test_index", "status", "size")); RelNode plan = makeLimit(scan, 10); @@ -80,7 +82,8 @@ public void testPureLimit_2shard() { """ OpenSearchSort(fetch=[10], viableBackends=[[mock-parquet]]) OpenSearchExchangeReducer(viableBackends=[[mock-parquet]], exchange=[ExchangeInfo[distributionType=SINGLETON, partitionKeyIndices=[]]]) - OpenSearchTableScan(table=[[test_index]], viableBackends=[[mock-parquet]]) + OpenSearchSort(fetch=[10], viableBackends=[[mock-parquet]]) + OpenSearchTableScan(table=[[test_index]], viableBackends=[[mock-parquet]]) """, result ); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPushdownPlanShapeTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPushdownPlanShapeTests.java index 2b8c042fdd0b8..8c461e6455dd4 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPushdownPlanShapeTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/SortPushdownPlanShapeTests.java @@ -37,6 +37,79 @@ private RelNode fetchLiteralSort(RelNode input, int fetch) { ); } + /** A collation-free LIMIT — PPL {@code head N} / SQL {@code LIMIT N} with no {@code ORDER BY}. */ + private RelNode bareLimit(RelNode input, int fetch) { + return LogicalSort.create( + input, + RelCollations.EMPTY, + null, + rexBuilder.makeLiteral(fetch, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + } + + /** + * Bare {@code head N} (no ORDER BY): a collation-free fetch must still be pushed below the ER so + * each shard caps locally at N rows. Without this, every shard streams its entire scan to + * the coordinator, which then trims to N — fetching every row defeats the limit. A limit without an + * order is always safe to push: the coordinator's N-row result is a subset of the union of each + * shard's N-row result, regardless of which rows each shard keeps. + */ + public void testBareLimit_2shard_pushed() { + RelNode scan = stubScan(mockTable("test_index", "status", "size")); + RelNode result = runPlanner(bareLimit(scan, 10), multiShardContext()); + assertPlanShape( + """ + OpenSearchSort(fetch=[10], viableBackends=[[mock-parquet]]) + OpenSearchExchangeReducer(viableBackends=[[mock-parquet]], exchange=[ExchangeInfo[distributionType=SINGLETON, partitionKeyIndices=[]]]) + OpenSearchSort(fetch=[10], viableBackends=[[mock-parquet]]) + OpenSearchTableScan(table=[[test_index]], viableBackends=[[mock-parquet]]) + """, + result + ); + } + + /** Bare LIMIT, single shard: no ER to push below — coordinator keeps the only Sort. */ + public void testBareLimit_singleShard_notPushed() { + RelNode scan = stubScan(mockTable("test_index", "status", "size")); + RelNode result = runPlanner(bareLimit(scan, 10), singleShardContext()); + assertEquals(1, RelOptUtil.toString(result).lines().filter(l -> l.contains("OpenSearchSort")).count()); + } + + /** Bare LIMIT with OFFSET: shard fetch widens to offset+fetch, offset stays on the coordinator. */ + public void testBareLimitOffset_2shard_widenedFetch() { + RelNode scan = stubScan(mockTable("test_index", "status", "size")); + RelNode plan = LogicalSort.create( + scan, + RelCollations.EMPTY, + rexBuilder.makeLiteral(5, typeFactory.createSqlType(SqlTypeName.INTEGER), true), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + String p = RelOptUtil.toString(runPlanner(plan, multiShardContext())); + assertEquals("coord + shard Sort", 2, p.lines().filter(l -> l.contains("OpenSearchSort")).count()); + assertTrue("coordinator keeps offset=5", p.contains("offset=[5]")); + int erIdx = p.indexOf("ExchangeReducer"); + int widenedIdx = p.indexOf("fetch=[15]"); + assertTrue("shard Sort widened to offset+fetch=15 below ER", erIdx >= 0 && widenedIdx > erIdx); + assertFalse("shard Sort must not carry an offset", p.substring(widenedIdx).contains("offset=")); + } + + /** Bare LIMIT over UNION ALL: the collation-free fetch is pushed below each arm's ER. */ + public void testBareLimitUnionAll_2shard_pushedIntoBothArms() { + RelNode union = LogicalUnion.create( + List.of(stubScan(mockTable("test_index", "status", "size")), stubScan(mockTable("test_index", "status", "size"))), + true + ); + String p = RelOptUtil.toString(runPlanner(bareLimit(union, 10), unionContext("test_index", 2))); + assertEquals("coordinator + one Sort per arm, plan:\n" + p, 3, p.lines().filter(l -> l.contains("OpenSearchSort")).count()); + assertEquals("one ER per arm, plan:\n" + p, 2, p.lines().filter(l -> l.contains("ExchangeReducer")).count()); + String[] lines = p.split("\n", -1); + for (int i = 0; i + 1 < lines.length; i++) { + if (lines[i].contains("ExchangeReducer")) { + assertTrue("a Sort must be pushed below each arm ER, plan:\n" + p, lines[i + 1].contains("OpenSearchSort")); + } + } + } + /** SQL shape: single Sort(collation, fetch) → identical Sort pushed below ER. * (A LateMaterialization node wraps the top for scans with stored fields — incidental.) */ public void testSqlSortLimit_2shard_pushed() { From 56abdaa5fec19aa50cd01f11958e124ef17c6213 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Sun, 7 Jun 2026 02:28:27 -0700 Subject: [PATCH 06/11] Fix: TopK rewriter must honor sort.offset in shard-fetch sizing (#22034) Signed-off-by: Marc Handalian --- .../planner/rules/OpenSearchTopKRewriter.java | 14 ++- .../planner/TopKRewriterPlanShapeTests.java | 98 ++++++++++++++++++- 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java index a2f580f3e6334..b57b52b80fa0a 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTopKRewriter.java @@ -60,7 +60,19 @@ public static Optional rewrite(RelNode root, PlannerContext context) { double factor = resolveOversamplingFactor(context); if (factor <= 0.0) return Optional.empty(); - long coordLimit = (sort.fetch instanceof RexLiteral lit) ? RexLiteral.intValue(lit) : 10_000L; + // coordLimit is the rank the coordinator must satisfy after merging shard streams: for + // `head N from M` it skips M then takes N, so the merged stream needs the global top-(M+N). + // Non-literal offsets bail to no-pushdown (PPL only emits literal offsets). + long fetch = (sort.fetch instanceof RexLiteral lit) ? RexLiteral.intValue(lit) : 10_000L; + long offset; + if (sort.offset == null) { + offset = 0L; + } else if (sort.offset instanceof RexLiteral lit) { + offset = RexLiteral.intValue(lit); + } else { + return Optional.empty(); + } + long coordLimit = offset + fetch; long shardSize = (long) Math.ceil(coordLimit * factor) + coordLimit; if (shardSize > Integer.MAX_VALUE) return Optional.empty(); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java index c114d678c8996..7923a5ec1b3ba 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/TopKRewriterPlanShapeTests.java @@ -16,6 +16,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; @@ -129,7 +130,7 @@ public void testRewrite_collatedOuterLimit_honorsInnerFetch() { String plan = RelOptUtil.toString(result); long sortCount = plan.lines().filter(l -> l.contains("OpenSearchSort")).count(); assertTrue("a per-partition Sort must be inserted", sortCount >= 2); - // Shard Sort fetch = ceil(innerFetch * factor) + innerFetch = ceil(10*2)+10 = 30. + // Shard Sort fetch = offset + ceil(fetch * factor) + fetch = 0 + ceil(10*2) + 10 = 30. assertTrue("shard Sort must be sized off the inner fetch (30), got plan:\n" + plan, plan.contains("fetch=[30]")); assertFalse("shard Sort must NOT be sized off the 10000 system cap (30000)", plan.contains("fetch=[30000]")); } @@ -223,6 +224,101 @@ public void testRewrite_multiGroupByCount_splitAndTopK() { ); } + /** + * {@code head 10 from 1000}: PPL emits {@code Sort(collation, offset=1000, fetch=10)}. The + * coordinator skips 1000 then takes 10, so its merged stream must contain the global top-1010 + * — i.e. {@code coordLimit = offset + fetch}. Shard fetch is then the usual oversampling + * formula applied to that coord limit: {@code ceil(coordLimit * factor) + coordLimit}. + * + *

    Expected shard fetch: {@code ceil(1010*2) + 1010 = 3030}. Without offset handling the + * rewriter used {@code coordLimit = fetch = 10}, so each shard shipped 30 rows, the + * coordinator skipped 1000, and the result was empty. + */ + public void testRewrite_sortWithOffset_shardFetchHonorsOffset() { + RelOptTable table = mockTable("test_index", "status", "size"); + RelNode scan = stubScan(table); + LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(0), null, List.of(countStarCall())); + RelNode sort = LogicalSort.create( + agg, + RelCollations.of(new RelFieldCollation(1, RelFieldCollation.Direction.DESCENDING)), + rexBuilder.makeLiteral(1000, typeFactory.createSqlType(SqlTypeName.INTEGER), true), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RelNode result = runPlanner(sort, contextWithOversampling(2.0)); + String plan = RelOptUtil.toString(result); + assertTrue( + "shard Sort must be sized as ceil((offset+fetch)*factor) + (offset+fetch) = 3030, got plan:\n" + plan, + plan.contains("fetch=[3030]") + ); + // Pre-fix bug: shard Sort sized off bare fetch (30) — coordinator's skip-1000 would have emptied the result. + assertFalse("shard Sort must NOT be sized off fetch alone (30)", plan.contains("fetch=[30]")); + } + + /** + * factor=1.0 with offset: oversampling collapses to {@code 2 * coordLimit}. For + * {@code head 10 from 1000} → 2*1010 = 2020. coordLimit = offset+fetch is still honored; + * factor=1 just removes the per-coord-row padding multiplier. + */ + public void testRewrite_factorOne_twiceCoordLimit() { + RelOptTable table = mockTable("test_index", "status", "size"); + RelNode scan = stubScan(table); + LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(0), null, List.of(countStarCall())); + RelNode sort = LogicalSort.create( + agg, + RelCollations.of(new RelFieldCollation(1, RelFieldCollation.Direction.DESCENDING)), + rexBuilder.makeLiteral(1000, typeFactory.createSqlType(SqlTypeName.INTEGER), true), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RelNode result = runPlanner(sort, contextWithOversampling(1.0)); + String plan = RelOptUtil.toString(result); + assertTrue("factor=1 → shard fetch = 2 * (offset+fetch) = 2020, got plan:\n" + plan, plan.contains("fetch=[2020]")); + } + + /** + * Offset close to {@code Integer.MAX_VALUE} pushes shardSize past int range — rewriter must + * bail (no per-partition Sort inserted). Without the bail an int overflow would produce a + * negative or wrong fetch literal. + */ + public void testRewrite_offsetOverflow_bails() { + RelOptTable table = mockTable("test_index", "status", "size"); + RelNode scan = stubScan(table); + LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(0), null, List.of(countStarCall())); + RelNode sort = LogicalSort.create( + agg, + RelCollations.of(new RelFieldCollation(1, RelFieldCollation.Direction.DESCENDING)), + rexBuilder.makeLiteral(Integer.MAX_VALUE - 5, typeFactory.createSqlType(SqlTypeName.INTEGER), true), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RelNode result = runPlanner(sort, contextWithOversampling(2.0)); + String plan = RelOptUtil.toString(result); + long sortCount = plan.lines().filter(l -> l.contains("OpenSearchSort")).count(); + assertEquals("overflow → no per-partition Sort inserted", 1, sortCount); + } + + /** + * Non-literal offset (parameterized expression): rewriter bails to no-pushdown rather than + * guessing a value. Each shard ships everything; correct but slow. PPL emits literal offsets + * in practice, so this is a defensive check. + */ + public void testRewrite_nonLiteralOffset_bails() { + RelOptTable table = mockTable("test_index", "status", "size"); + RelNode scan = stubScan(table); + LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(0), null, List.of(countStarCall())); + RexNode lit5 = rexBuilder.makeLiteral(5, typeFactory.createSqlType(SqlTypeName.INTEGER), true); + RexNode lit10 = rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true); + RexNode nonLiteralOffset = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, lit5, lit10); + RelNode sort = LogicalSort.create( + agg, + RelCollations.of(new RelFieldCollation(1, RelFieldCollation.Direction.DESCENDING)), + nonLiteralOffset, + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RelNode result = runPlanner(sort, contextWithOversampling(2.0)); + String plan = RelOptUtil.toString(result); + long sortCount = plan.lines().filter(l -> l.contains("OpenSearchSort")).count(); + assertEquals("non-literal offset → no per-partition Sort inserted", 1, sortCount); + } + // ── Helpers ────────────────────────────────────────────────────────────── private RelNode buildSortHeadOverGroupedCount() { From 2f078b6615ea68484850751d2083fcb09352b785 Mon Sep 17 00:00:00 2001 From: expani1729 <110471048+expani@users.noreply.github.com> Date: Sun, 7 Jun 2026 02:33:43 -0700 Subject: [PATCH 07/11] Fix performance-delegation demotion in filter delegation and remove fuse_dual_viable setting (#22035) Signed-off-by: Aniketh Jain --- .../rust/src/indexed_table/bool_tree.rs | 20 +- ...DelegationForIndexFullConversionTests.java | 2 +- .../LuceneAnalyticsBackendPluginTests.java | 2 +- .../opensearch/analytics/AnalyticsPlugin.java | 20 - .../analytics/exec/DefaultPlanExecutor.java | 5 +- .../dag/DelegatedPredicateCombiner.java | 91 +++-- .../planner/dag/FilterTreeShapeDeriver.java | 87 +++-- .../planner/dag/FragmentConversionDriver.java | 31 +- .../planner/dag/DAGBuilderTests.java | 4 +- .../dag/FilterTreeShapeDeriverTests.java | 81 +--- .../dag/FragmentConversionDriverTests.java | 358 +++++++----------- .../be/datafusion/QtfSubstraitDumpIT.java | 4 +- .../analytics/qa/CountFastPathIT.java | 4 +- .../qa/FilterDelegationGoldenIT.java | 239 ++++-------- .../analytics/qa/FilterDelegationIT.java | 89 ++--- .../opensearch/analytics/qa/QueryCacheIT.java | 6 +- 16 files changed, 394 insertions(+), 649 deletions(-) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs index 5665ebc75266f..548386fb04c3f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs @@ -183,18 +183,16 @@ impl BoolNode { } } - /// Demote every `DelegationPossible` leaf to a plain `Predicate` carrying its - /// `original_expr`. Called when the filter classifies as `FilterClass::Tree` - /// (i.e. has any OR or NOT in its shape) — the bitmap-tree evaluator currently - /// has `unimplemented!()` arms for `DelegationPossible`, so any disjunctive query - /// containing a performance-delegated leaf would otherwise crash the data node. + /// Demote every `DelegationPossible` leaf to a plain native `Predicate`. Called for + /// `FilterClass::Tree` plans (any OR/NOT), whose evaluator can't run `DelegationPossible`. + /// The coordinator never puts perf directly under OR/NOT, but an AND-side perf leaf can ride + /// in a tree that still has a surviving OR/NOT (e.g. `tag=a AND (dual OR native)`); demoting + /// it to native keeps results correct, just skipping the opportunistic peer consult. /// - /// Demotion is a uniform, position-blind transform: every DelegationPossible in - /// the tree becomes a Predicate, even ones under all-AND ancestry within the - /// tree. That sacrifices the AND-side perf-delegation opportunity for those - /// queries — acceptable v1 trade-off — in exchange for "doesn't crash." - /// Position-aware demotion (only OR/NOT-positioned leaves) would need - /// DelegationPossible support in the bitmap-tree evaluator. + /// FIXME: move this Tree-classification awareness into the coordinator so it never emits a + /// `delegation_possible` that reaches a Tree plan; then this method goes away and the + /// evaluator arms can throw. Pairs with implementing performance delegation under OR/NOT in + /// the Tree-path evaluator. pub fn demote_delegation_possible(self) -> BoolNode { match self { BoolNode::And(children) => { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java index dbdb754464177..764616916414d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java @@ -244,7 +244,7 @@ private StagePlan runPipeline(RexNode condition) { RelNode marked = PlannerImpl.runAllOptimizations(filter, context); QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); Stage leaf = dag.rootStage(); while (!leaf.getChildStages().isEmpty()) { diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java index a4074f55d9d82..700b6d39d0748 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java @@ -154,7 +154,7 @@ public void testMatchPredicateDelegationEndToEnd() throws IOException { PlanForker.forkAll(dag, context.getCapabilityRegistry()); BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry()); PlanAlternativeSelector.selectAll(dag, context.getCapabilityRegistry(), false); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); // Find the leaf stage (shard scan with filter) Stage leaf = dag.rootStage(); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java index d3282c3ff7a75..4a64ea9e16e54 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java @@ -107,25 +107,6 @@ public class AnalyticsPlugin extends Plugin implements ExtensiblePlugin, ActionP Setting.Property.Dynamic ); - /** - * When {@code true} (default), performance-delegated leaves (driver natively evaluable, - * peer also viable) fuse with their correctness-delegated siblings even under {@code OR} - * / {@code NOT}. The combiner ships the entire boolean structure as a single delegated - * expression rather than throwing the dual-viable leaves back to native. - * - *

    Default {@code true} — Lucene's term-dictionary random access typically beats - * managing per-leaf bitsets in DataFusion, so fusing the OR/NOT into one peer call is - * the favorable choice for the common workload. Flip to {@code false} for A/B comparison - * or to roll back if a workload regresses (e.g. very wide OR over highly-selective - * leaves where the driver's column scan would short-circuit before Lucene completes). - */ - public static final Setting DELEGATION_FUSE_DUAL_VIABLE = Setting.boolSetting( - "analytics.delegation.fuse_dual_viable", - true, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - /** * Controls the metadata-only driver vs. value-producing peer choice when both are viable * for a stage: @@ -255,7 +236,6 @@ public Collection createGuiceModules() { public List> getSettings() { List> settings = new java.util.ArrayList<>(); settings.add(COORDINATOR_BUFFER_LIMIT); - settings.add(DELEGATION_FUSE_DUAL_VIABLE); settings.add(PREFER_METADATA_DRIVER); settings.add(ReaderContextStore.READER_CONTEXT_KEEP_ALIVE); settings.addAll(org.opensearch.analytics.settings.AnalyticsApproximationSettings.all()); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index 48c721381160f..77761d314d739 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -94,7 +94,6 @@ public class DefaultPlanExecutor extends HandledTransportAction maxConcurrentShardRequestsPerNode = v ); - this.fuseDualViable = AnalyticsPlugin.DELEGATION_FUSE_DUAL_VIABLE.get(clusterService.getSettings()); - clusterService.getClusterSettings().addSettingsUpdateConsumer(AnalyticsPlugin.DELEGATION_FUSE_DUAL_VIABLE, v -> fuseDualViable = v); this.preferMetadataDriver = AnalyticsPlugin.PREFER_METADATA_DRIVER.get(clusterService.getSettings()); clusterService.getClusterSettings() .addSettingsUpdateConsumer(AnalyticsPlugin.PREFER_METADATA_DRIVER, v -> preferMetadataDriver = v); @@ -248,7 +245,7 @@ private void executeInternal( // Collapse multi-backend stages to a single chosen alternative before conversion // so the convertor runs once per stage and the wire request carries one PlanAlternative. PlanAlternativeSelector.selectAll(dag, capabilityRegistry, preferMetadataDriver); - FragmentConversionDriver.convertAll(dag, capabilityRegistry, fuseDualViable); + FragmentConversionDriver.convertAll(dag, capabilityRegistry); final long planningTimeNanos = System.nanoTime() - planStartNanos; final long planningTimeMs = profile ? java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(planningTimeNanos) : 0; logger.debug("[DefaultPlanExecutor] QueryDAG:\n{}", dag); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DelegatedPredicateCombiner.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DelegatedPredicateCombiner.java index 91f83d3d70185..67970fd405155 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DelegatedPredicateCombiner.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DelegatedPredicateCombiner.java @@ -45,34 +45,33 @@ final class DelegatedPredicateCombiner { private final CapabilityRegistry registry; private final RexBuilder rexBuilder; private final List delegatedExpressions; - /** - * When true, dual-viable leaves are classified as correctness-delegated everywhere — - * the {@code delegation_possible} marker (driver-evaluates-natively + opportunistic peer - * consult) is not emitted under fuse=true. Result: one merged {@code delegated_predicate} - * ships the whole eligible subtree to the peer; the driver doesn't evaluate any - * delegatable arm itself. Trade-off: AND-side opportunistic optimization gone in - * exchange for cross-bucket merging under OR/NOT with native siblings. - */ - private final boolean fuseDualViable; DelegatedPredicateCombiner( String operatorBackend, List fieldStorage, CapabilityRegistry registry, RexBuilder rexBuilder, - List delegatedExpressions, - boolean fuseDualViable + List delegatedExpressions ) { this.operatorBackend = operatorBackend; this.fieldStorage = fieldStorage; this.registry = registry; this.rexBuilder = rexBuilder; this.delegatedExpressions = delegatedExpressions; - this.fuseDualViable = fuseDualViable; } /** Bottom-up: classify each node as Delegated (carries the RexNode subtree) or Resolved. */ Classified classify(RexNode node, Function applyFn) { + return classify(node, applyFn, false); + } + + /** + * @param underOrNot true when any ancestor is an OR/NOT. Performance delegation can't survive a + * disjunction, so a dual-viable leaf anywhere under an OR/NOT — even nested in + * an AND — is reclassified to correctness in combine(). Threaded so this + * happens before a mixed inner AND materializes and hides the perf marker. + */ + private Classified classify(RexNode node, Function applyFn, boolean underOrNot) { if (node instanceof AnnotatedPredicate ap) { String backend = ap.getViableBackends().getFirst(); if (!backend.equals(operatorBackend) && canSerialize(ap, backend)) { @@ -80,10 +79,13 @@ Classified classify(RexNode node, Function applyFn) } else if (!ap.getPerformanceDelegationBackends().isEmpty()) { String peerBackend = ap.getPerformanceDelegationBackends().getFirst(); if (canSerialize(ap, peerBackend)) { - // Under fuseDualViable, demote perf to correctness so the leaf merges - // freely with correctness siblings and ships entirely to the peer. - boolean isPerf = !fuseDualViable; - return new Delegated(peerBackend, node, ap.getAnnotationId(), isPerf); + // Dual-viable leaf → always performance-delegated. Demotion to correctness is + // decided only by tree position in combine() (under OR/NOT), never here. + // TODO: an AND-side perf leaf can still ride in a tree with a surviving OR/NOT + // (e.g. tag=a AND (dual OR native)); the data node Tree-classifies the whole + // tree and demotes this marker to native. Detecting that here would let the + // data-node fallback be removed. + return new Delegated(peerBackend, node, ap.getAnnotationId(), true); } } return new Resolved(applyFn.apply(ap)); @@ -95,19 +97,23 @@ Classified classify(RexNode node, Function applyFn) return new Resolved(resolveCallChildren(call, applyFn)); } + // Monotonic: once under an OR/NOT, every descendant is too (any depth of nested AND). + boolean childUnderOrNot = underOrNot || kind == SqlKind.OR || kind == SqlKind.NOT; List kids = new ArrayList<>(call.getOperands().size()); for (RexNode operand : call.getOperands()) { - kids.add(classify(operand, applyFn)); + kids.add(classify(operand, applyFn, childUnderOrNot)); } - return combine(call, kids, applyFn); + return combine(call, kids, applyFn, underOrNot); } return new Resolved(node); } /** Combines classified children of an AND/OR/NOT call into a single Classified result. */ - private Classified combine(RexCall call, List kids, Function applyFn) { - boolean isOrNot = call.getKind() == SqlKind.OR || call.getKind() == SqlKind.NOT; + private Classified combine(RexCall call, List kids, Function applyFn, boolean underOrNot) { + // True under any OR/NOT (including an AND nested beneath one): perf can't survive a + // disjunction, so perf children here are carved to correctness. + boolean isOrNot = underOrNot || call.getKind() == SqlKind.OR || call.getKind() == SqlKind.NOT; List correctnessChildren = new ArrayList<>(); List performanceChildren = new ArrayList<>(); @@ -116,21 +122,27 @@ private Classified combine(RexCall call, List kids, Function kids, Function kids, FunctionPerformance delegation composes only under AND, so a perf leaf doesn't survive under OR/NOT: + * under OR it's reclassified to correctness (ships to the peer); under NOT it stays native + * (NOT(=) folds to != with no serializer). The shape label must match what the combiner emits, + * or the data node mis-routes (the historical "all-docs" disjunction bug). + */ + private static Result walk(RexNode node, String drivingBackendId) { if (node instanceof AnnotatedPredicate predicate) { - // Two flavors of delegation count toward "hasDelegated": - // 1. Correctness — viableBackends differs from operator backend (the only backend - // that can evaluate is the peer). - // 2. Performance — operator backend can evaluate natively, but a peer was also - // viable and is available for opportunistic per-RG consultation. boolean isCorrectness = !predicate.getViableBackends().getFirst().equals(drivingBackendId); boolean isPerformance = !predicate.getPerformanceDelegationBackends().isEmpty(); - boolean isDelegated = isCorrectness || isPerformance; - return new Result(isDelegated, false, !isDelegated, isPerformance); + boolean isDrivingBackend = !isCorrectness && !isPerformance; + return new Result(isCorrectness, isPerformance, isDrivingBackend, false); } if (node instanceof RexCall call) { - boolean isOrNot = call.getKind() == SqlKind.OR || call.getKind() == SqlKind.NOT; + boolean isOr = call.getKind() == SqlKind.OR; + boolean isNot = call.getKind() == SqlKind.NOT; - boolean hasDelegated = false; + boolean hasCorrectness = false; + boolean hasPerf = false; boolean hasDrivingBackend = false; - boolean hasPerformanceDelegation = false; - boolean hasMixed = false; - + boolean interleaved = false; for (RexNode operand : call.getOperands()) { - Result childResult = walk(operand, drivingBackendId, fuseDualViable); - hasDelegated |= childResult.hasDelegated; - hasDrivingBackend |= childResult.hasDrivingBackend; - hasMixed |= childResult.hasMixed; - hasPerformanceDelegation |= childResult.hasPerformanceDelegation; + Result child = walk(operand, drivingBackendId); + hasCorrectness |= child.hasCorrectness; + hasPerf |= child.hasPerf; + hasDrivingBackend |= child.hasDrivingBackend; + interleaved |= child.interleaved; + } + + // The data node does performance delegation only on the AND path, so a perf leaf can't + // stay perf under OR or NOT. Mirror combine(): + if (hasPerf && isOr) { + // Under OR it's reclassified to correctness and shipped to the peer. + hasCorrectness = true; + hasPerf = false; + } else if (hasPerf && isNot) { + // Under NOT a dual-equality leaf folds to != (no serializer) and stays native. + hasDrivingBackend = true; + hasPerf = false; } - // Under OR/NOT, interleaving occurs when delegated children sit alongside something - // the combiner can't fuse with them: - // - native (driving-backend) operands — never fuse, always interleaved. - // - performance-delegated operands — interleaved only when fuseDualViable is off - // (carve-out throws perf back to native individually); with fusion on the - // combiner emits a single delegation_possible wrapping the whole OR/NOT. - boolean unfuseablePeer = hasDrivingBackend || (hasPerformanceDelegation && fuseDualViable == false); - hasMixed |= isOrNot && hasDelegated && unfuseablePeer; + // Interleaving (tree evaluator needed) arises under OR/NOT when a delegated shipment + // sits alongside a driving-backend operand — the two can't collapse into one peer + // shipment. Under AND, driving-backend + delegated coexist as separate conjuncts + // (single-collector path), so AND never introduces interleaving on its own. + interleaved |= (isOr || isNot) && hasCorrectness && hasDrivingBackend; - return new Result(hasDelegated, hasMixed, hasDrivingBackend, hasPerformanceDelegation); + return new Result(hasCorrectness, hasPerf, hasDrivingBackend, interleaved); } return new Result(false, false, false, false); } - private record Result(boolean hasDelegated, boolean hasMixed, boolean hasDrivingBackend, boolean hasPerformanceDelegation) { + private record Result(boolean hasCorrectness, boolean hasPerf, boolean hasDrivingBackend, boolean interleaved) { } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java index 158ed206810f5..e44445ed7eb54 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java @@ -80,13 +80,9 @@ private FragmentConversionDriver() {} /** * Converts all {@link StagePlan} alternatives in the DAG, populating * {@link StagePlan#convertedBytes()} on each plan. - * - * @param fuseDualViable when {@code true}, performance-delegated leaves fuse with - * correctness-delegated siblings even under OR/NOT. Sourced from the cluster setting - * {@code analytics.delegation.fuse_dual_viable}. */ - public static void convertAll(QueryDAG dag, CapabilityRegistry registry, boolean fuseDualViable) { - convertStage(dag.rootStage(), registry, fuseDualViable); + public static void convertAll(QueryDAG dag, CapabilityRegistry registry) { + convertStage(dag.rootStage(), registry); // Root stage executes locally at coordinator — store factory for instruction dispatch. Stage root = dag.rootStage(); if (root.getExchangeSinkProvider() != null && !root.getPlanAlternatives().isEmpty()) { @@ -95,9 +91,9 @@ public static void convertAll(QueryDAG dag, CapabilityRegistry registry, boolean } } - private static void convertStage(Stage stage, CapabilityRegistry registry, boolean fuseDualViable) { + private static void convertStage(Stage stage, CapabilityRegistry registry) { for (Stage child : stage.getChildStages()) { - convertStage(child, registry, fuseDualViable); + convertStage(child, registry); } // After children are converted, surface any decorator-induced schema delta as // postDecorationSchemaBytes on the child plans. The reduce sink consults this when @@ -115,15 +111,15 @@ private static void convertStage(Stage stage, CapabilityRegistry registry, boole AnalyticsSearchBackendPlugin backend = registry.getBackend(plan.backendId()); FragmentConvertor convertor = backend.getFragmentConvertor(); - // Derive filter tree shape BEFORE stripping (annotations must be intact). Mirrors - // fuseDualViable so the deriver's classification matches the post-combiner tree - // the data node actually sees. + // Derive filter tree shape BEFORE stripping (annotations must be intact). The deriver + // mirrors the combiner's post-combine shape so the data node's classification matches + // the tree it actually receives. OpenSearchFilter filter = RelNodeUtils.findNode(plan.resolvedFragment(), OpenSearchFilter.class); FilterTreeShape treeShape = filter != null - ? FilterTreeShapeDeriver.derive(filter, plan.backendId(), fuseDualViable) + ? FilterTreeShapeDeriver.derive(filter, plan.backendId()) : FilterTreeShape.NO_DELEGATION; - IntraOperatorDelegationBytes delegationBytes = new IntraOperatorDelegationBytes(registry, fuseDualViable); + IntraOperatorDelegationBytes delegationBytes = new IntraOperatorDelegationBytes(registry); byte[] bytes = convert(plan.resolvedFragment(), convertor, delegationBytes); // Assemble instruction list @@ -279,16 +275,10 @@ private static boolean isPureReorderProject(org.apache.calcite.rel.core.Project */ static final class IntraOperatorDelegationBytes { private final CapabilityRegistry registry; - private final boolean fuseDualViable; private List delegatedExpressions; IntraOperatorDelegationBytes(CapabilityRegistry registry) { - this(registry, false); - } - - IntraOperatorDelegationBytes(CapabilityRegistry registry, boolean fuseDualViable) { this.registry = registry; - this.fuseDualViable = fuseDualViable; } /** @@ -305,8 +295,7 @@ Function resolverFor(OpenSearchRelNode operator, Re fieldStorage, registry, rexBuilder, - delegatedExpressions, - fuseDualViable + delegatedExpressions ); return new AnnotationResolver() { diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/DAGBuilderTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/DAGBuilderTests.java index 76df4af1f389e..e0a4140bee379 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/DAGBuilderTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/DAGBuilderTests.java @@ -334,7 +334,7 @@ public FragmentConvertor getFragmentConvertor() { QueryDAG dag = DAGBuilder.build(cbo, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); LOGGER.info("QueryDAG:\n{}", dag); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); // LM at root with no above-ops: the LM stage IS the root — no synthetic post-LM stage. assertEquals( @@ -406,7 +406,7 @@ public FragmentConvertor getFragmentConvertor() { QueryDAG dag = DAGBuilder.build(cbo, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); LOGGER.info("QueryDAG:\n{}", dag); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); // Stage 3 has real compute (the Project) on top of the StageInputScan → COORDINATOR_REDUCE. assertEquals( diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java index 174ec6ca182d0..aca534d50ef00 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java @@ -32,7 +32,7 @@ public void testNoDelegation() { RexNode nativePred = annotated(DRIVING); OpenSearchFilter filter = buildFilter(nativePred); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals("No delegation should return PLAIN", FilterTreeShape.NO_DELEGATION, shape); } @@ -43,7 +43,7 @@ public void testSingleDelegatedPredicate() { RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, delegated); OpenSearchFilter filter = buildFilter(andNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.CONJUNCTIVE, shape); } @@ -55,7 +55,7 @@ public void testMultipleDelegatedUnderAnd() { RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, delegated1, delegated2); OpenSearchFilter filter = buildFilter(andNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.CONJUNCTIVE, shape); } @@ -66,7 +66,7 @@ public void testOrWithDelegatedAndNative() { RexNode orNode = rexBuilder.makeCall(SqlStdOperatorTable.OR, nativePred, delegated); OpenSearchFilter filter = buildFilter(orNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); } @@ -78,7 +78,7 @@ public void testNotWithDelegated() { RexNode notNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT, andNode); OpenSearchFilter filter = buildFilter(notNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); } @@ -92,7 +92,7 @@ public void testOrWithOnlyDelegated() { RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, orNode); OpenSearchFilter filter = buildFilter(andNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.CONJUNCTIVE, shape); } @@ -103,7 +103,7 @@ public void testBareNotOfDelegated() { RexNode notNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT, delegated); OpenSearchFilter filter = buildFilter(notNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.CONJUNCTIVE, shape); } @@ -123,15 +123,18 @@ private AnnotatedPredicate perfDelegated() { return (AnnotatedPredicate) dualViable.narrowTo(DRIVING); } + // Delegated OR Dual: the Dual leaf can't stay performance-based under OR (perf delegation is + // AND-only in the data node), so it's demoted to correctness and fused with the Delegated leaf + // into ONE Lucene shipment. The shape is then a single delegated leaf → label = CONJUNCTIVE. + // (The label must match what the combiner emits; a mismatch here is what caused the all-docs bug.) public void testOrWithCorrectnessAndPerfDelegated() { - // OR(correctness-delegated, perf-delegated) — perf won't combine under OR → INTERLEAVED RexNode correctness = annotated(ACCEPTING); RexNode perf = perfDelegated(); RexNode orNode = rexBuilder.makeCall(SqlStdOperatorTable.OR, correctness, perf); OpenSearchFilter filter = buildFilter(orNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); - assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.CONJUNCTIVE, shape); } public void testAndWithCorrectnessAndPerfDelegated() { @@ -141,67 +144,19 @@ public void testAndWithCorrectnessAndPerfDelegated() { RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, correctness, perf); OpenSearchFilter filter = buildFilter(andNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); assertEquals(FilterTreeShape.CONJUNCTIVE, shape); } + // NOT(dual-equality) is not delegated — it stays native → NO_DELEGATION. (Combiner counterpart: + // FragmentConversionDriverTests#testNotEqualsStaysNative, NOT(EQUALS) → 0 delegated expressions.) public void testNotOfPerfDelegated() { - // NOT(perf-delegated) — perf under NOT won't combine → INTERLEAVED RexNode perf = perfDelegated(); RexNode notNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT, perf); OpenSearchFilter filter = buildFilter(notNode); - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, false); - assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); - } - - // ---- fuse_dual_viable interaction ---- - - /** - * Same OR(correctness, perf) shape as {@link #testOrWithCorrectnessAndPerfDelegated} but - * with {@code fuseDualViable=true}: combiner fuses both delegated children (correctness ∪ - * perf) into a single {@code delegated_predicate} placeholder wrapping the OR. The peer - * evaluates the whole boolean (driver can't decompose OR into independently-evaluable - * arms — {@code delegation_possible} only composes under AND). Post-combiner tree is a - * bare delegated leaf — conjunctive from the data-node evaluator's perspective. - */ - public void testOrWithCorrectnessAndPerfDelegated_fused_isConjunctive() { - RexNode correctness = annotated(ACCEPTING); - RexNode perf = perfDelegated(); - RexNode orNode = rexBuilder.makeCall(SqlStdOperatorTable.OR, correctness, perf); - OpenSearchFilter filter = buildFilter(orNode); - - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, true); - assertEquals(FilterTreeShape.CONJUNCTIVE, shape); - } - - /** - * NOT(perf-delegated) with {@code fuseDualViable=true}: combiner keeps the perf leaf in - * the delegation pool under NOT, so the post-combiner tree is a single - * {@code delegation_possible} wrapping the NOT — conjunctive. - */ - public void testNotOfPerfDelegated_fused_isConjunctive() { - RexNode perf = perfDelegated(); - RexNode notNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT, perf); - OpenSearchFilter filter = buildFilter(notNode); - - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, true); - assertEquals(FilterTreeShape.CONJUNCTIVE, shape); - } - - /** - * Even with {@code fuseDualViable=true}, OR(delegated, native-non-delegable) stays - * INTERLEAVED — the native arm can't be folded into the peer's delegation no matter - * what the carve-out policy is. Fusion only collapses the perf-vs-correctness axis. - */ - public void testOrWithDelegatedAndNative_fused_stillInterleaved() { - RexNode delegated = annotated(ACCEPTING); - RexNode nativePred = annotated(DRIVING); - RexNode orNode = rexBuilder.makeCall(SqlStdOperatorTable.OR, delegated, nativePred); - OpenSearchFilter filter = buildFilter(orNode); - - FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING, true); - assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.NO_DELEGATION, shape); } private OpenSearchFilter buildFilter(RexNode condition) { diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java index 58fa68abd6114..67b8f006e2520 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java @@ -111,7 +111,7 @@ private QueryDAG buildAndConvert(int shardCount, RelNode logicalPlan, RecordingC LOGGER.info("Marked+CBO:\n{}", RelOptUtil.toString(cboOutput)); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); LOGGER.info("QueryDAG after conversion:\n{}", dag); return dag; } @@ -322,7 +322,7 @@ public org.opensearch.analytics.spi.FragmentConvertor getFragmentConvertor() { LOGGER.info("Marked+CBO:\n{}", RelOptUtil.toString(cboOutput)); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); Stage root = dag.rootStage(); assertNotNull("root alternative must have convertedBytes", root.getPlanAlternatives().getFirst().convertedBytes()); @@ -523,26 +523,6 @@ private QueryDAG buildDelegationDag( String[] fieldNames, SqlTypeName[] fieldTypes, Map> fields - ) { - return buildDelegationDag(condition, dfConvertor, serializer, fieldNames, fieldTypes, fields, false); - } - - /** - * Like the non-fused overload but threads {@code fuseDualViable} through to - * {@link FragmentConversionDriver#convertAll(QueryDAG, CapabilityRegistry, boolean)} so - * tests can flip the {@code analytics.delegation.fuse_dual_viable} cluster setting - * deterministically. With {@code fuse=true}, performance-delegated leaves under OR/NOT - * stay in the delegation pool instead of being thrown back to native — the entire boolean - * ships to the peer as one delegated expression. - */ - private QueryDAG buildDelegationDag( - RexNode condition, - RecordingConvertor dfConvertor, - RecordingSerializer serializer, - String[] fieldNames, - SqlTypeName[] fieldTypes, - Map> fields, - boolean fuseDualViable ) { var backends = delegationBackends(dfConvertor, serializer); var context = buildContext("parquet", fields, backends); @@ -551,7 +531,7 @@ private QueryDAG buildDelegationDag( LOGGER.info("Marked+CBO:\n{}", RelOptUtil.toString(cboOutput)); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), fuseDualViable); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); return dag; } @@ -577,16 +557,6 @@ private QueryDAG buildSingleFieldDelegationDag(RexNode condition, RecordingConve * - field 2 = amount (integer, NOT indexed) — SINGLE-VIABLE to DataFusion only. */ private QueryDAG buildTwoFieldDelegationDag(RexNode condition, RecordingConvertor dfConvertor, RecordingSerializer serializer) { - return buildTwoFieldDelegationDag(condition, dfConvertor, serializer, false); - } - - /** {@link #buildTwoFieldDelegationDag} with the {@code fuse_dual_viable} setting threaded through. */ - private QueryDAG buildTwoFieldDelegationDag( - RexNode condition, - RecordingConvertor dfConvertor, - RecordingSerializer serializer, - boolean fuseDualViable - ) { return buildDelegationDag( condition, dfConvertor, @@ -600,8 +570,7 @@ private QueryDAG buildTwoFieldDelegationDag( Map.of("type", "keyword", "index", true), "amount", Map.of("type", "integer", "index", false) - ), - fuseDualViable + ) ); } @@ -611,12 +580,7 @@ private QueryDAG buildTwoFieldDelegationDag( * Mirrors prod Lucene's {@code STANDARD_TYPES = {KEYWORD, TEXT, MATCH_ONLY_TEXT}} — * EQUALS-on-int is NOT a Lucene cap in prod even when the field has BKD points. */ - private QueryDAG buildKeywordPlusNativeDag( - RexNode condition, - RecordingConvertor dfConvertor, - RecordingSerializer serializer, - boolean fuseDualViable - ) { + private QueryDAG buildKeywordPlusNativeDag(RexNode condition, RecordingConvertor dfConvertor, RecordingSerializer serializer) { return buildDelegationDag( condition, dfConvertor, @@ -630,8 +594,7 @@ private QueryDAG buildKeywordPlusNativeDag( Map.of("type", "keyword", "index", true), "amount", Map.of("type", "integer", "index", false) - ), - fuseDualViable + ) ); } @@ -839,7 +802,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); // Should produce 1 combined DelegatedExpression (not 2 individual ones) @@ -899,7 +862,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); // OR siblings targeting same backend should be combined into 1 DelegatedExpression @@ -960,7 +923,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); assertEquals("Pure Lucene nested OR(AND,leaf) should produce 1 expression", 1, plan.delegatedExpressions().size()); @@ -1012,7 +975,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); assertEquals("OR(AND,AND) pure Lucene should produce 1 expression", 1, plan.delegatedExpressions().size()); @@ -1065,7 +1028,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); assertEquals("AND(OR(AND,leaf),leaf) pure Lucene should produce 1 expression", 1, plan.delegatedExpressions().size()); @@ -1130,7 +1093,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); // All 4 Lucene predicates combined into 1, native amount=200 stays separate @@ -1221,7 +1184,7 @@ public Map delegatedPredicateSeria RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); return new CombiningResult(plan, dfConvertor, serializer); } @@ -1408,127 +1371,83 @@ public void testInterleavedAndOrNot() { assertTrue("AND structure should be preserved", strippedPlan.contains("AND")); } - // ---- fuse_dual_viable cluster setting ---- + // ---- Dual-viable leaves stay performance-based by default ---- /** - * Two performance-delegated leaves plus a correctness-delegated one under OR, with the - * {@code analytics.delegation.fuse_dual_viable} cluster setting governing how the perf - * leaves are emitted. - * - *

      - *
    • {@code fuse=false} — combiner's OR/NOT carve-out throws each performance-delegated - * leaf back individually: each becomes its own {@code delegation_possible} marker in - * the driver plan AND its own entry in {@code delegatedExpressions}. So 3 leaves → - * 3 expressions and 2 separate {@code delegation_possible} markers (one per perf - * leaf), the correctness one converts via the combine path.
      • - *
    • {@code fuse=true} — combiner aggregates both perf leaves into a single perf - * bucket and the correctness leaf into one correctness bucket. The perf bucket - * emits ONE {@code delegation_possible} marker wrapping the combined perf subtree. - * Total: 2 delegated expressions, 1 {@code delegation_possible}, 1 - * {@code delegated_predicate}.
      • - *
    + * Dual AND Dual: both leaves stay performance-based. The two same-backend perf leaves fuse into + * one shipped expression; the plan carries delegation_possible (DataFusion prunes natively + + * opportunistic peer consult) and NO delegated_predicate (nothing shipped wholesale to the peer). * - *

    Setup: {@code status} is index=true (indexed integer) and {@code message} is keyword - * indexed — EQUALS on both is dual-viable (performance-delegation candidate). FUZZY on - * message is correctness-delegated to Lucene. Two distinct fields are used so Calcite's - * SEARCH/Sarg rewrite doesn't fold the two EQUALS into a single sargable predicate. + *

    Regression guard for the perf-demotion bug: previously every dual-viable leaf was demoted to + * correctness-based delegation and shipped wholesale to Lucene — even under a plain AND where + * nothing forces it — so DataFusion never evaluated it. A demoted result here would show a bare + * delegated_predicate and zero delegation_possible. */ - public void testOrPerformanceLeavesAndCorrectness_fuseDualViable_explicitFalse() { - var ctx = buildFuseTestSetup(); - QueryDAG dag = buildKeywordFuseDag(ctx.condition, ctx.dfConvertor, ctx.serializer, /* fuseDualViable */ false); - StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - - assertEquals("fuse=false: 3 delegated expressions (2 perf leaves + 1 correctness)", 3, plan.delegatedExpressions().size()); - String strippedPlan = RelOptUtil.toString(ctx.dfConvertor.shardScanFragment); - // Two separate delegation_possible markers — one per individually thrown-back perf leaf. - assertEquals( - "fuse=false: 2 separate delegation_possible markers in plan", - 2, - countOccurrences(strippedPlan, "delegation_possible") + public void testAndTwoDual_defaultsToPerformanceDelegation() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + // tag, message both keyword(indexed) → dual-viable for EQUALS; amount int(not indexed) = native. + RexNode tagEq = rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0), + rexBuilder.makeLiteral("alpha") ); - assertTrue("OR structure preserved", strippedPlan.contains("OR")); - } - - public void testOrPerformanceLeavesAndCorrectness_fuseDualViable_explicitTrue() { - var ctx = buildFuseTestSetup(); - QueryDAG dag = buildKeywordFuseDag(ctx.condition, ctx.dfConvertor, ctx.serializer, /* fuseDualViable */ true); + RexNode msgEq = rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 1), + rexBuilder.makeLiteral("hello") + ); + QueryDAG dag = buildKeywordPlusNativeDag(makeAnd(tagEq, msgEq), dfConvertor, serializer); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + String stripped = RelOptUtil.toString(dfConvertor.shardScanFragment); - // Under OR with mixed correctness+perf children, the driver can't decompose the boolean - // (delegation_possible only composes under AND). Combiner fuses ALL delegated children - // into a single delegated_predicate placeholder — peer evaluates the whole OR. - // Post-combiner shape: delegated_predicate(...) — one bare collector at top. - assertEquals( - "fuse=true under OR: 1 delegated expression (correctness + perf fused into one peer subtree)", - 1, - plan.delegatedExpressions().size() - ); - String strippedPlan = RelOptUtil.toString(ctx.dfConvertor.shardScanFragment); - // No delegation_possible markers — OR with mixed kinds collapses to a single - // correctness-style placeholder so the data-node classifier sees a bare Collector - // (CONJUNCTIVE → SingleCollector evaluator), not OR(delegated, delegation_possible). - assertEquals( - "fuse=true under OR with correctness sibling: 0 delegation_possible markers", - 0, - countOccurrences(strippedPlan, "delegation_possible") + assertTrue( + "Dual AND Dual must stay performance-based (delegation_possible present). Plan: " + stripped, + stripped.contains(DelegationPossibleFunction.NAME) ); - assertEquals( - "fuse=true: exactly 1 delegated_predicate marker at the top of the post-combiner tree", - 1, - countOccurrences(strippedPlan, "delegated_predicate") + assertFalse( + "Dual AND Dual must NOT be demoted to correctness (no delegated_predicate). Plan: " + stripped, + stripped.contains(DelegatedPredicateFunction.NAME) ); } + // ---- OR with mixed correctness + performance delegation ---- + /** - * Randomized: picks {@code fuse} at random, asserts the discriminating invariant — - * perf-leaf count in the AST changes with the setting. Surfaces non-obvious combiner - * regressions across both branches over many test runs. + * {@code OR(perf-tag, perf-message, FUZZY-correctness)} — two dual-viable leaves plus a + * correctness-delegated leaf, all same-backend, under OR. Performance delegation can't survive + * a disjunction, and a correctness sibling is present, so the combiner fuses the entire OR into + * ONE {@code delegated_predicate} shipped to the peer: 1 delegated expression, 1 + * delegated_predicate marker, 0 delegation_possible markers. */ - public void testOrPerformanceLeavesAndCorrectness_fuseDualViable_randomized() { - boolean fuse = randomBoolean(); + public void testOrPerformanceLeavesAndCorrectness_fusesToOneDelegated() { var ctx = buildFuseTestSetup(); - QueryDAG dag = buildKeywordFuseDag(ctx.condition, ctx.dfConvertor, ctx.serializer, fuse); + QueryDAG dag = buildKeywordFuseDag(ctx.condition, ctx.dfConvertor, ctx.serializer); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); String strippedPlan = RelOptUtil.toString(ctx.dfConvertor.shardScanFragment); - int delegationPossibleMarkers = countOccurrences(strippedPlan, "delegation_possible"); - int delegatedPredicateMarkers = countOccurrences(strippedPlan, "delegated_predicate"); - if (fuse) { - // Under OR with mixed correctness+perf, fuse=true collapses the entire boolean to - // ONE delegated_predicate (peer evaluates whole OR). No delegation_possible because - // the driver can't decompose OR into independently-evaluable arms. - assertEquals("fuse=true: 0 delegation_possible markers (OR with correctness sibling fuses all)", 0, delegationPossibleMarkers); - assertEquals("fuse=true: 1 delegated_predicate marker at top of post-combiner tree", 1, delegatedPredicateMarkers); - assertEquals( - "fuse=true: 1 delegated expression (correctness + perf fused into one peer subtree)", - 1, - plan.delegatedExpressions().size() - ); - } else { - assertEquals("fuse=false: perf leaves carved out individually → 2 delegation_possible markers", 2, delegationPossibleMarkers); - assertEquals("fuse=false: 3 delegated expressions (2 perf leaves + correctness)", 3, plan.delegatedExpressions().size()); - } + assertEquals("1 delegated expression (whole OR ships as one peer subtree)", 1, plan.delegatedExpressions().size()); + assertEquals( + "0 delegation_possible markers — performance delegation doesn't compose under OR", + 0, + countOccurrences(strippedPlan, "delegation_possible") + ); + assertEquals("exactly 1 delegated_predicate marker at top", 1, countOccurrences(strippedPlan, "delegated_predicate")); } /** - * Regression: the exact prod bug shape — 2-leaf {@code OR(MATCH-correctness, EQUALS-perf)}. - * Pre-fix, the combiner Mixed branch emitted - * {@code OR(delegated_predicate(matchId), delegation_possible(EQUALS, eqId))}; the data-node - * Java deriver returned CONJUNCTIVE, the Rust classifier override forced SingleCollector, - * and {@code single_collector_id} on the OR top returned None — the residual extraction - * fell into the OR-passthrough arm and evaluated as {@code true} for every row, returning - * 100% match instead of the actual count (observed end-to-end on the ClickBench dataset: - * fuse=true returned 99,997,497 = total docs, fuse=false returned 28,313,573). - * - *

    Post-fix the combiner under OR/NOT with mixed correctness+perf same-backend children - * collapses both into one {@code delegated_predicate} placeholder. Post-combiner shape is a - * bare delegated leaf — Rust's {@code is_and_only_collector_tree} accepts it, and the - * SingleCollector evaluator routes the whole BoolQuery to Lucene. + * Regression for the prod "all-docs" disjunction bug — 2-leaf {@code OR(MATCH-correctness, + * EQUALS-perf)}. Previously the combiner emitted {@code OR(delegated_predicate(matchId), + * delegation_possible(EQUALS, eqId))} while the tree-shape deriver labelled it CONJUNCTIVE; the + * data node trusted the label, forced the single-collector path onto a tree containing an OR, + * failed to apply the filter, and returned every row (ClickBench: 99,997,497 vs. the correct + * 28,313,573). * - *

    This test pins the post-combiner shape: under fuse=true, exactly 1 delegated_predicate - * marker, zero delegation_possible markers, exactly 1 DelegatedExpression entry. + *

    Now the combiner collapses an OR/NOT with a same-backend correctness sibling into one + * {@code delegated_predicate} — a bare delegated leaf the data node routes wholesale to Lucene. + * Pins: exactly 1 delegated_predicate marker, zero delegation_possible markers, 1 expression. */ - public void testOrCorrectnessPlusPerf_twoLeaves_fuseDualViable_collapsesToOneDelegated() { + public void testOrCorrectnessPlusPerf_twoLeaves_collapsesToOneDelegated() { RecordingConvertor dfConvertor = new RecordingConvertor(); RecordingSerializer serializer = new RecordingSerializer(); // OR(MATCH(message,'hello') correctness-delegated, EQUALS(tag='alpha') perf-delegated) @@ -1541,52 +1460,44 @@ public void testOrCorrectnessPlusPerf_twoLeaves_fuseDualViable_collapsesToOneDel rexBuilder.makeLiteral("alpha") ) ); - QueryDAG dag = buildKeywordFuseDag(condition, dfConvertor, serializer, /* fuseDualViable */ true); + QueryDAG dag = buildKeywordFuseDag(condition, dfConvertor, serializer); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); String strippedPlan = RelOptUtil.toString(dfConvertor.shardScanFragment); assertEquals( - "fuse=true 2-leaf OR(correctness, perf): 1 delegated expression (whole OR ships as one peer subtree)", + "2-leaf OR(correctness, perf): 1 delegated expression (whole OR ships as one peer subtree)", 1, plan.delegatedExpressions().size() ); assertEquals( - "fuse=true 2-leaf OR(correctness, perf): 0 delegation_possible markers — " - + "delegation_possible doesn't compose with disjunction", + "2-leaf OR(correctness, perf): 0 delegation_possible markers — doesn't compose under disjunction", 0, countOccurrences(strippedPlan, "delegation_possible") ); assertEquals( - "fuse=true 2-leaf OR(correctness, perf): exactly 1 delegated_predicate marker at top", + "2-leaf OR(correctness, perf): exactly 1 delegated_predicate marker at top", 1, countOccurrences(strippedPlan, "delegated_predicate") ); } /** - * Combiner shape matrix across AND/OR/NOT × fuse modes × native sibling presence. - * Each row pins the post-combiner shape via three counters: + * Combiner shape matrix across AND/OR/NOT × native-sibling presence. Each row pins the + * post-combiner shape via three counters: *

      - *
    1. number of {@code DelegatedExpression} entries the data node receives,
      2. - *
    2. number of {@code delegated_predicate(annotationId)} markers in the rebuilt - * RexCall (peer evaluates these subtrees end-to-end),
      3. - *
    3. number of {@code delegation_possible(original, annotationId)} markers (driver - * evaluates the wrapped predicate natively, peer consulted opportunistically).
      4. + *
    4. {@code DelegatedExpression} entries the data node receives,
      5. + *
    5. {@code delegated_predicate(id)} markers (peer evaluates these end-to-end),
      6. + *
    6. {@code delegation_possible(original, id)} markers (driver evaluates natively, peer + * consulted opportunistically).
      7. *
    * - *

    fuse=true contract: dual-viable leaves classify as correctness everywhere, so - * {@code delegation_possible} never appears under fuse=true. Cross-bucket merging - * (correctness + dual-viable in the same boolean) collapses to a single - * {@code delegated_predicate} regardless of native siblings. - * - *

    fuse=false contract: dual-viable leaves stay as performance-delegated under AND - * (one {@code delegation_possible} per leaf) but get carved back to native individually - * under OR/NOT. Native-only siblings stay native at the top of the rebuilt boolean. + *

    Rule: a dual-viable leaf is performance-delegated under AND (one delegation_possible per + * leaf); under OR/NOT it's reclassified to correctness, fusing with any same-backend correctness + * siblings into one delegated_predicate. Native-only siblings stay native at the top. * *

    Field setup (from {@code buildKeywordPlusNativeDag}, prod-Lucene-shaped): - * tag:keyword(index=true) and message:keyword(index=true) → both dual-viable for EQUALS - * (perf-delegated to Lucene); message also handles MATCH/FUZZY (Lucene-correctness); - * amount:int(index=false) → no Lucene format, native-only. + * tag/message:keyword(index=true) → dual-viable for EQUALS; message also handles MATCH/FUZZY + * (Lucene-correctness); amount:int(index=false) → native-only. */ public void testCombinerShapeMatrix_andOrNot_withAndWithoutNativeSibling() { // ── shapes ────────────────────────────────────────────────────────── @@ -1609,64 +1520,68 @@ public void testCombinerShapeMatrix_andOrNot_withAndWithoutNativeSibling() { // amount has index=false → no Lucene format → native-only. RexNode nativeArm = makeEquals(2, SqlTypeName.INTEGER, 42); - // (label, condition, fuse=false expectation, fuse=true expectation) + // (label, condition, expectation) — expectation = {delegatedExpressions, + // delegated_predicate markers, delegation_possible markers}. Single expectation per shape: + // delegation is now decided purely by tree position. + // + // Rules: a dual-viable (perf) leaf stays performance-delegated (delegation_possible) under + // AND. Under OR/NOT it cannot stay perf (perf delegation is AND-only in the data node), so + // it's reclassified to correctness and ships to the peer, fusing with same-backend siblings. Object[][] cases = { - // ── AND: dual-viable stays as delegation_possible per-leaf under fuse=false ── - { "AND(MATCH, EQUALS-perf)", makeAnd(match, eqPerf), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "AND(MATCH, EQUALS-perf, native)", makeAnd(match, eqPerf, nativeArm), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "AND(MATCH, native)", makeAnd(match, nativeArm), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, - - // ── OR: dual-viable carves back to native under fuse=false ───────── - { "OR(MATCH, FUZZY)", or(match, fuzzy), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, - { "OR(EQUALS-perf, EQUALS-perf2)", or(eqPerf, eqPerf2), new int[] { 2, 0, 2 }, new int[] { 1, 1, 0 } }, - { "OR(EQUALS-perf, EQUALS-perf2, native)", or(eqPerf, eqPerf2, nativeArm), new int[] { 2, 0, 2 }, new int[] { 1, 1, 0 } }, - { "OR(MATCH, EQUALS-perf)", or(match, eqPerf), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "OR(MATCH, EQUALS-perf, native)", or(match, eqPerf, nativeArm), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "OR(MATCH, FUZZY, native)", or(match, fuzzy, nativeArm), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, - { "OR(MATCH, native)", or(match, nativeArm), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, + // ── AND: perf stays perf (delegation_possible per perf leaf) ── + { "AND(MATCH, EQUALS-perf)", makeAnd(match, eqPerf), new int[] { 2, 1, 1 } }, + { "AND(MATCH, EQUALS-perf, native)", makeAnd(match, eqPerf, nativeArm), new int[] { 2, 1, 1 } }, + { "AND(MATCH, native)", makeAnd(match, nativeArm), new int[] { 1, 1, 0 } }, + + // ── OR perf-only: perf reclassified to correctness, fused into one peer shipment ── + { "OR(EQUALS-perf, EQUALS-perf2)", or(eqPerf, eqPerf2), new int[] { 1, 1, 0 } }, + { "OR(EQUALS-perf, EQUALS-perf2, native)", or(eqPerf, eqPerf2, nativeArm), new int[] { 1, 1, 0 } }, + + // ── OR/NOT with a same-backend correctness sibling: perf fuses into it → 1 delegated_predicate ── + { "OR(MATCH, FUZZY)", or(match, fuzzy), new int[] { 1, 1, 0 } }, + { "OR(MATCH, EQUALS-perf)", or(match, eqPerf), new int[] { 1, 1, 0 } }, + { "OR(MATCH, EQUALS-perf, native)", or(match, eqPerf, nativeArm), new int[] { 1, 1, 0 } }, + { "OR(MATCH, FUZZY, native)", or(match, fuzzy, nativeArm), new int[] { 1, 1, 0 } }, + { "OR(MATCH, native)", or(match, nativeArm), new int[] { 1, 1, 0 } }, // ── NOT shapes (Calcite folds NOT(=) → ≠ pre-combiner, so no perf survives there) ── - { "NOT(MATCH)", rexBuilder.makeCall(SqlStdOperatorTable.NOT, match), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, - { "NOT(EQUALS-perf)", rexBuilder.makeCall(SqlStdOperatorTable.NOT, eqPerf), new int[] { 0, 0, 0 }, new int[] { 0, 0, 0 } }, + { "NOT(MATCH)", rexBuilder.makeCall(SqlStdOperatorTable.NOT, match), new int[] { 1, 1, 0 } }, + { "NOT(EQUALS-perf)", rexBuilder.makeCall(SqlStdOperatorTable.NOT, eqPerf), new int[] { 0, 0, 0 } }, - // ── NOT inside boolean — the prod-bug shape from the OR-fuse fix ─── - { "OR(NOT(MATCH), EQUALS-perf)", or(notMatch, eqPerf), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "OR(NOT(MATCH), EQUALS-perf, native)", or(notMatch, eqPerf, nativeArm), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "AND(NOT(MATCH), EQUALS-perf, native)", makeAnd(notMatch, eqPerf, nativeArm), new int[] { 2, 1, 1 }, new int[] { 1, 1, 0 } }, - { "OR(NOT(MATCH), native)", or(notMatch, nativeArm), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, - { "AND(NOT(MATCH), native)", makeAnd(notMatch, nativeArm), new int[] { 1, 1, 0 }, new int[] { 1, 1, 0 } }, }; + // ── NOT inside boolean — the prod-bug shape from the OR-disjunction fix ─── + { "OR(NOT(MATCH), EQUALS-perf)", or(notMatch, eqPerf), new int[] { 1, 1, 0 } }, + { "OR(NOT(MATCH), EQUALS-perf, native)", or(notMatch, eqPerf, nativeArm), new int[] { 1, 1, 0 } }, + { "AND(NOT(MATCH), EQUALS-perf, native)", makeAnd(notMatch, eqPerf, nativeArm), new int[] { 2, 1, 1 } }, + { "OR(NOT(MATCH), native)", or(notMatch, nativeArm), new int[] { 1, 1, 0 } }, + { "AND(NOT(MATCH), native)", makeAnd(notMatch, nativeArm), new int[] { 1, 1, 0 } }, }; for (Object[] testCase : cases) { String label = (String) testCase[0]; RexNode condition = (RexNode) testCase[1]; - int[] expectedFalse = (int[]) testCase[2]; - int[] expectedTrue = (int[]) testCase[3]; - assertCombinerShape(label, condition, false, expectedFalse[0], expectedFalse[1], expectedFalse[2]); - assertCombinerShape(label, condition, true, expectedTrue[0], expectedTrue[1], expectedTrue[2]); + int[] expected = (int[]) testCase[2]; + assertCombinerShape(label, condition, expected[0], expected[1], expected[2]); } } private void assertCombinerShape( String label, RexNode condition, - boolean fuse, int expDelegatedExprs, int expDelegatedPredicate, int expDelegationPossible ) { RecordingConvertor c = new RecordingConvertor(); - QueryDAG d = buildKeywordPlusNativeDag(condition, c, new RecordingSerializer(), fuse); + QueryDAG d = buildKeywordPlusNativeDag(condition, c, new RecordingSerializer()); StagePlan p = leafStage(d).getPlanAlternatives().getFirst(); String stripped = RelOptUtil.toString(c.shardScanFragment); - String prefix = label + " fuse=" + fuse; - assertEquals(prefix + " — delegatedExpressions (plan: " + stripped + ")", expDelegatedExprs, p.delegatedExpressions().size()); + assertEquals(label + " — delegatedExpressions (plan: " + stripped + ")", expDelegatedExprs, p.delegatedExpressions().size()); assertEquals( - prefix + " — delegated_predicate markers (plan: " + stripped + ")", + label + " — delegated_predicate markers (plan: " + stripped + ")", expDelegatedPredicate, countOccurrences(stripped, "delegated_predicate") ); assertEquals( - prefix + " — delegation_possible markers (plan: " + stripped + ")", + label + " — delegation_possible markers (plan: " + stripped + ")", expDelegationPossible, countOccurrences(stripped, "delegation_possible") ); @@ -1703,22 +1618,16 @@ private FuseTestSetup buildFuseTestSetup() { private record FuseTestSetup(RecordingConvertor dfConvertor, RecordingSerializer serializer, RexNode condition) { } - /** Two-keyword-field delegation helper for the fuse tests. Both fields are dual-viable; - * no integer field is involved, so this stays consistent with prod Lucene caps. */ - private QueryDAG buildKeywordFuseDag( - RexNode condition, - RecordingConvertor dfConvertor, - RecordingSerializer serializer, - boolean fuseDualViable - ) { + /** Two-keyword-field delegation helper. Both fields are dual-viable; no integer field is + * involved, so this stays consistent with prod Lucene caps. */ + private QueryDAG buildKeywordFuseDag(RexNode condition, RecordingConvertor dfConvertor, RecordingSerializer serializer) { return buildDelegationDag( condition, dfConvertor, serializer, new String[] { "tag", "message" }, new SqlTypeName[] { SqlTypeName.VARCHAR, SqlTypeName.VARCHAR }, - Map.of("tag", Map.of("type", "keyword", "index", true), "message", Map.of("type", "keyword", "index", true)), - fuseDualViable + Map.of("tag", Map.of("type", "keyword", "index", true), "message", Map.of("type", "keyword", "index", true)) ); } @@ -1776,7 +1685,7 @@ public FragmentConvertor getFragmentConvertor() { PlanForker.forkAll(dag, context.getCapabilityRegistry()); IllegalStateException exception = expectThrows( IllegalStateException.class, - () -> FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false) + () -> FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()) ); assertTrue(exception.getMessage().contains("No DelegatedPredicateSerializer")); } @@ -1820,7 +1729,7 @@ public FragmentConvertor getFragmentConvertor() { RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); // Only MATCH_PHRASE delegated; EQUALS stays native (no serializer) assertEquals("Only serializable predicates delegated", 1, plan.delegatedExpressions().size()); @@ -1867,7 +1776,7 @@ public FragmentConvertor getFragmentConvertor() { RelNode cboOutput = runPlanner(filter, context); QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); // Correctness and performance delegated stay separate — 2 DelegatedExpressions assertEquals("Correctness and performance delegated stay separate", 2, plan.delegatedExpressions().size()); @@ -1953,8 +1862,9 @@ public void testAndOnlyPerfAndNative() { } /** - * match(Title,'ru') OR Referer='google' AND URL='US' AND GoodEvent=1 - * Under OR, perf-delegated stays native (resolved via applyFn). + * match(Title,'ru') OR (Referer='google' AND URL='US' AND GoodEvent=1) + * Under OR the Referer/URL perf leaves ship to Lucene (reclassified to correctness), so the plan + * carries delegated_predicate but no delegation_possible. GoodEvent is native. */ public void testOrCorrectnessWithPerfAndNative() { RecordingConvertor dfConvertor = new RecordingConvertor(); @@ -1986,12 +1896,11 @@ public void testOrCorrectnessWithPerfAndNative() { ) ); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - // match is correctness-delegated (1 expression), perf under AND already combined (1 expression) assertEquals(2, plan.delegatedExpressions().size()); String strippedPlan = RelOptUtil.toString(dfConvertor.shardScanFragment); LOGGER.info("Plan (OR correctness with perf+native):\n{}", strippedPlan); - assertTrue("Should have delegated_predicate for match", strippedPlan.contains(DelegatedPredicateFunction.NAME)); - assertTrue("Should have delegation_possible for perf", strippedPlan.contains(DelegationPossibleFunction.NAME)); + assertTrue("Should have delegated_predicate", strippedPlan.contains(DelegatedPredicateFunction.NAME)); + assertFalse("perf ships to Lucene under OR — no delegation_possible", strippedPlan.contains(DelegationPossibleFunction.NAME)); assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, treeShapeOf(plan)); } @@ -2030,7 +1939,10 @@ public void testAndCorrectnessOrPerfWithNative() { String strippedPlan = RelOptUtil.toString(dfConvertor.shardScanFragment); LOGGER.info("Plan (AND correctness OR perf + native):\n{}", strippedPlan); assertTrue("Should have delegated_predicate", strippedPlan.contains(DelegatedPredicateFunction.NAME)); - assertTrue("Should have delegation_possible", strippedPlan.contains(DelegationPossibleFunction.NAME)); + assertFalse( + "perf can't survive under OR — must not have delegation_possible", + strippedPlan.contains(DelegationPossibleFunction.NAME) + ); assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, treeShapeOf(plan)); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java index 3925f86dbbad2..cd4bf4653c613 100644 --- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/QtfSubstraitDumpIT.java @@ -140,7 +140,7 @@ public void testDumpQtfPipeline() throws Exception { LOGGER.info("[QTF-DUMP] QueryDAG (pre-conversion):\n{}", dag); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); LOGGER.info("[QTF-DUMP] QueryDAG (post-conversion, with backend-resolved fragments):\n{}", dag); // Walk every stage and dump its substrait Plan(s). @@ -213,7 +213,7 @@ private QueryDAG buildAndConvertQtfDag(String sql) { RelNode cbo = PlannerImpl.runAllOptimizations(parsed, context); QueryDAG dag = DAGBuilder.build(cbo, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); PlanForker.forkAll(dag, context.getCapabilityRegistry()); - FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry(), false); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); return dag; } diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CountFastPathIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CountFastPathIT.java index 2298e3f7f1b0e..c64733e2c23c2 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CountFastPathIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CountFastPathIT.java @@ -183,8 +183,8 @@ public void testKeywordBooleanFilter_luceneFused() throws Exception { oracleWhere(d -> d.userID.equals("u_a") && tokenizedContains(d.message, "alpha")) ); - // OR mixing keyword EQUALS with MATCH — the combiner's OR/NOT carve-out behavior - // depends on the fuse_dual_viable setting; both modes must produce the same count. + // OR mixing keyword EQUALS with MATCH — the dual-viable EQUALS fuses with the MATCH + // correctness sibling into one Lucene shipment under OR; the count is unaffected. assertCount( "where userID = 'u_b' OR match(message, 'gamma') | stats count() as cnt", oracleWhere(d -> d.userID.equals("u_b") || tokenizedContains(d.message, "gamma")) diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationGoldenIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationGoldenIT.java index 700cd7cc339fe..5d80ff9258240 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationGoldenIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationGoldenIT.java @@ -16,10 +16,10 @@ import java.util.Map; /** - * Filter-delegation matrix IT. Walks every enabled {@link Shape} through the - * {@code (prefer_metadata_driver × fuse_dual_viable)} 4-cell matrix, asserts response - * equality (against {@code ppl/expected/q{N}.json}) and per-cell {@code chosen_backend} / - * {@code tree_shape} on the SHARD_FRAGMENT profile. + * Filter-delegation matrix IT. Walks every enabled {@link Shape} through both + * {@code prefer_metadata_driver} values, asserts response equality (against + * {@code ppl/expected/q{N}.json}) and per-cell {@code chosen_backend} / {@code tree_shape} + * on the SHARD_FRAGMENT profile. * *

    Leaf vocabulary: Dual (DataFusion + Lucene, e.g. keyword EQUALS), * Native (DataFusion only, e.g. long EQUALS), Delegated (Lucene only, @@ -52,10 +52,9 @@ protected void onBeforeQuery() throws IOException { @Override public void tearDown() throws Exception { - // Restore defaults so a test that toggled {prefer,fuse} doesn't leak into the next. + // Restore default so a test that toggled prefer_metadata_driver doesn't leak into the next. try { setPreferMetadataDriver(true); - setFuseDualViable(true); } finally { super.tearDown(); } @@ -119,190 +118,108 @@ public void tearDown() throws Exception { // ===================================================================== /** - * Per-cell matrix for {@code (prefer_metadata_driver, fuse_dual_viable)}. - * - *

    Cell args are in the order: {@code (prefer=true,fuse=false)}, - * {@code (true,true)}, {@code (false,false)}, {@code (false,true)}. - * - *

    {@link ChosenBackendandTreeShape#placeholder()} disables the stage assertion for that cell - * (the response oracle still runs). Used for shapes whose query throws before - * producing a profile (the 4 known-red bug shapes). + * Per-shape expected (chosen_backend, tree_shape), one cell per {@code prefer_metadata_driver} + * value: {@code (preferTrue, preferFalse)}. */ private enum Shape { + // Cells are (prefer_metadata_driver=true, prefer=false). prefer=true lets Lucene drive the + // whole stage when every arm is Lucene-viable (chosen=lucene, no tree_shape); a native arm + // forces datafusion. prefer=false runs the combiner, where delegation shape is decided by + // tree position: a dual-viable leaf stays performance-delegated under AND; under OR/NOT it's + // reclassified to correctness and ships to Lucene (fusing with same-backend correctness + // siblings). A delegated shipment beside a native arm under OR is INTERLEAVED; otherwise CONJUNCTIVE. + // Single leaf (3) - SINGLE_DUAL(1, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - SINGLE_NATIVE(2, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - SINGLE_DELEGATED(3, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), + SINGLE_DUAL(1, lucene(), df("CONJUNCTIVE")), + SINGLE_NATIVE(2, df(null), df(null)), + SINGLE_DELEGATED(3, lucene(), df("CONJUNCTIVE")), // Two-leaf AND (6) - AND_DUAL_DUAL(4, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_NATIVE_NATIVE(5, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - AND_DELEGATED_DELEGATED(6, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_DUAL_NATIVE(7, - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_DUAL_DELEGATED(8, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_NATIVE_DELEGATED(9, - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), + AND_DUAL_DUAL(4, lucene(), df("CONJUNCTIVE")), + AND_NATIVE_NATIVE(5, df(null), df(null)), + AND_DELEGATED_DELEGATED(6, lucene(), df("CONJUNCTIVE")), + AND_DUAL_NATIVE(7, df("CONJUNCTIVE"), df("CONJUNCTIVE")), + AND_DUAL_DELEGATED(8, lucene(), df("CONJUNCTIVE")), + AND_NATIVE_DELEGATED(9, df("CONJUNCTIVE"), df("CONJUNCTIVE")), // Two-leaf OR (6) - OR_DUAL_DUAL(10, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - OR_NATIVE_NATIVE(11, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - OR_DELEGATED_DELEGATED(12, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - OR_DUAL_NATIVE(13, - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION")), - OR_DUAL_DELEGATED(14, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - OR_NATIVE_DELEGATED(15, - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION")), + OR_DUAL_DUAL(10, lucene(), df("CONJUNCTIVE")), + OR_NATIVE_NATIVE(11, df(null), df(null)), + OR_DELEGATED_DELEGATED(12, lucene(), df("CONJUNCTIVE")), + OR_DUAL_NATIVE(13, df("INTERLEAVED_BOOLEAN_EXPRESSION"), df("INTERLEAVED_BOOLEAN_EXPRESSION")), + OR_DUAL_DELEGATED(14, lucene(), df("CONJUNCTIVE")), + OR_NATIVE_DELEGATED(15, df("INTERLEAVED_BOOLEAN_EXPRESSION"), df("INTERLEAVED_BOOLEAN_EXPRESSION")), // Three-leaf AND (7) - AND_DUAL_DUAL_DUAL(16, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_NATIVE_NATIVE_NATIVE(17, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - AND_DELEGATED_DELEGATED_DELEGATED(18, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_DUAL_DUAL_DELEGATED(19, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_DUAL_DUAL_NATIVE(20, - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_DELEGATED_DELEGATED_NATIVE(21, - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - AND_DUAL_DELEGATED_NATIVE(22, - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), + AND_DUAL_DUAL_DUAL(16, lucene(), df("CONJUNCTIVE")), + AND_NATIVE_NATIVE_NATIVE(17, df(null), df(null)), + AND_DELEGATED_DELEGATED_DELEGATED(18, lucene(), df("CONJUNCTIVE")), + AND_DUAL_DUAL_DELEGATED(19, lucene(), df("CONJUNCTIVE")), + AND_DUAL_DUAL_NATIVE(20, df("CONJUNCTIVE"), df("CONJUNCTIVE")), + AND_DELEGATED_DELEGATED_NATIVE(21, df("CONJUNCTIVE"), df("CONJUNCTIVE")), + AND_DUAL_DELEGATED_NATIVE(22, df("CONJUNCTIVE"), df("CONJUNCTIVE")), // Three-leaf OR (7) - OR_DUAL_DUAL_DUAL(23, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - OR_NATIVE_NATIVE_NATIVE(24, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - OR_DELEGATED_DELEGATED_DELEGATED(25, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - OR_DUAL_DUAL_DELEGATED(26, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - OR_DUAL_DUAL_NATIVE(27, - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION")), - OR_DELEGATED_DELEGATED_NATIVE(28, - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION")), - OR_DUAL_DELEGATED_NATIVE(29, - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION")), + OR_DUAL_DUAL_DUAL(23, lucene(), df("CONJUNCTIVE")), + OR_NATIVE_NATIVE_NATIVE(24, df(null), df(null)), + OR_DELEGATED_DELEGATED_DELEGATED(25, lucene(), df("CONJUNCTIVE")), + OR_DUAL_DUAL_DELEGATED(26, lucene(), df("CONJUNCTIVE")), + OR_DUAL_DUAL_NATIVE(27, df("INTERLEAVED_BOOLEAN_EXPRESSION"), df("INTERLEAVED_BOOLEAN_EXPRESSION")), + OR_DELEGATED_DELEGATED_NATIVE(28, df("INTERLEAVED_BOOLEAN_EXPRESSION"), df("INTERLEAVED_BOOLEAN_EXPRESSION")), + OR_DUAL_DELEGATED_NATIVE(29, df("INTERLEAVED_BOOLEAN_EXPRESSION"), df("INTERLEAVED_BOOLEAN_EXPRESSION")), // NOT(leaf) (3) - NOT_DUAL(30, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - NOT_NATIVE(31, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)), - NOT_DELEGATED(32, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), + NOT_DUAL(30, df(null), df(null)), + NOT_NATIVE(31, df(null), df(null)), + NOT_DELEGATED(32, lucene(), df("CONJUNCTIVE")), // Mixed connectors, depth 2 (6) - MIXED_OR_OF_ANDS_OF_DUALS(33, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - MIXED_OR_OF_ANDS_OF_DELEGATED(34, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - MIXED_OR_OF_DUAL_DELEGATED_ANDS(35, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - MIXED_AND_OF_DUAL_DELEGATED_ORS(36, - new ChosenBackendandTreeShape("lucene", null), new ChosenBackendandTreeShape("lucene", null), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "CONJUNCTIVE")), - MIXED_OR_OF_AND_OF_DUALS_AND_NATIVE(37, - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), new ChosenBackendandTreeShape("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION")), - MIXED_NOT_OF_AND_OF_DUALS(38, - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null), - new ChosenBackendandTreeShape("datafusion", null), new ChosenBackendandTreeShape("datafusion", null)); + MIXED_OR_OF_ANDS_OF_DUALS(33, lucene(), df("CONJUNCTIVE")), + MIXED_OR_OF_ANDS_OF_DELEGATED(34, lucene(), df("CONJUNCTIVE")), + MIXED_OR_OF_DUAL_DELEGATED_ANDS(35, lucene(), df("CONJUNCTIVE")), + MIXED_AND_OF_DUAL_DELEGATED_ORS(36, lucene(), df("CONJUNCTIVE")), + MIXED_OR_OF_AND_OF_DUALS_AND_NATIVE(37, df("INTERLEAVED_BOOLEAN_EXPRESSION"), df("INTERLEAVED_BOOLEAN_EXPRESSION")), + MIXED_NOT_OF_AND_OF_DUALS(38, df(null), df(null)); final int queryNumber; - final Map cells; + final Map cells; - Shape(int queryNumber, - ChosenBackendandTreeShape preferTrue_fuseFalse, - ChosenBackendandTreeShape preferTrue_fuseTrue, - ChosenBackendandTreeShape preferFalse_fuseFalse, - ChosenBackendandTreeShape preferFalse_fuseTrue) { + Shape(int queryNumber, ChosenBackendandTreeShape preferTrue, ChosenBackendandTreeShape preferFalse) { this.queryNumber = queryNumber; - Map map = new LinkedHashMap<>(); - map.put(new SettingCombination(true, false), preferTrue_fuseFalse); - map.put(new SettingCombination(true, true), preferTrue_fuseTrue); - map.put(new SettingCombination(false, false), preferFalse_fuseFalse); - map.put(new SettingCombination(false, true), preferFalse_fuseTrue); + Map map = new LinkedHashMap<>(); + map.put(true, preferTrue); + map.put(false, preferFalse); this.cells = java.util.Collections.unmodifiableMap(map); } } + private static ChosenBackendandTreeShape lucene() { + return new ChosenBackendandTreeShape("lucene", null); + } + + private static ChosenBackendandTreeShape df(String treeShape) { + return new ChosenBackendandTreeShape("datafusion", treeShape); + } + // ===================================================================== // Driver / matrix harness // ===================================================================== - /** Cluster-setting combination: ({@code prefer_metadata_driver}, {@code fuse_dual_viable}). */ - private record SettingCombination(boolean prefer, boolean fuse) {} - /** Asserted SHARD_FRAGMENT profile fields. {@code treeShape == null} means the field - * must be absent (Lucene-as-driver has no delegation instruction). A {@code null} - * {@code chosenBackend} marks an unfilled placeholder cell — the harness will skip - * the stage assertions for that cell, but still validate the row oracle. */ - private record ChosenBackendandTreeShape(String chosenBackend, String treeShape) { - static ChosenBackendandTreeShape placeholder() { return new ChosenBackendandTreeShape(null, null); } - boolean isPlaceholder() { return chosenBackend == null; } - } + * must be absent (Lucene-as-driver, or no delegation instruction). */ + private record ChosenBackendandTreeShape(String chosenBackend, String treeShape) {} private void runShape(Shape shape) throws Exception { int queryNumber = shape.queryNumber; String ppl = DatasetProvisioner.loadResource(DATASET.queryResourcePath("ppl", "ppl", queryNumber)).trim(); ppl = ppl.replace(DATASET.name, DATASET.indexName); - for (Map.Entry entry : shape.cells.entrySet()) { - SettingCombination key = entry.getKey(); + for (Map.Entry entry : shape.cells.entrySet()) { + boolean prefer = entry.getKey(); ChosenBackendandTreeShape expected = entry.getValue(); - setPreferMetadataDriver(key.prefer()); - setFuseDualViable(key.fuse()); + setPreferMetadataDriver(prefer); - String label = shape + " prefer=" + key.prefer() + ",fuse=" + key.fuse(); + String label = shape + " prefer=" + prefer; // Profile=false path — guards against any profile-only-induced behavior change masking a regression. Map bareResponse = executePpl(ppl, false); @@ -318,11 +235,9 @@ private void runShape(Shape shape) throws Exception { fail(label + " (profile=true) — " + validationError); } - if (expected.isPlaceholder() == false) { - Map stage = shardFragmentStage(response); - assertEquals(label + " — chosen_backend", expected.chosenBackend(), stage.get("chosen_backend")); - assertEquals(label + " — tree_shape", expected.treeShape(), stage.get("tree_shape")); - } + Map stage = shardFragmentStage(response); + assertEquals(label + " — chosen_backend", expected.chosenBackend(), stage.get("chosen_backend")); + assertEquals(label + " — tree_shape", expected.treeShape(), stage.get("tree_shape")); } } @@ -358,12 +273,6 @@ private Map shardFragmentStage(Map response) { throw new AssertionError("No SHARD_FRAGMENT stage in profile: " + stages); } - private void setFuseDualViable(boolean value) throws Exception { - Request req = new Request("PUT", "/_cluster/settings"); - req.setJsonEntity("{\"persistent\":{\"analytics.delegation.fuse_dual_viable\": " + value + "}}"); - client().performRequest(req); - } - private void setPreferMetadataDriver(boolean value) throws Exception { Request req = new Request("PUT", "/_cluster/settings"); req.setJsonEntity("{\"persistent\":{\"analytics.planner.prefer_metadata_driver\": " + value + "}}"); diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java index 8daa131bc5c19..3e919801f89c9 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java @@ -295,71 +295,62 @@ public void testOrOfTwoAndArms() throws Exception { /** * OR(MATCH on text, EQUALS on keyword), oracle = 10. Both arms are Lucene-delegatable. * Under {@code prefer=true} Lucene drives end-to-end (combiner skipped, no tree_shape). - * Under {@code prefer=false} the combiner runs: {@code fuse=false} keeps - * {@code OR(delegated_predicate, delegation_possible)} as INTERLEAVED; {@code fuse=true} - * collapses to a single {@code delegated_predicate} as CONJUNCTIVE. + * Under {@code prefer=false} the combiner runs: the dual-viable EQUALS can't stay + * performance-delegated under OR, and it has a correctness sibling (the MATCH), so both + * collapse into a single {@code delegated_predicate} — CONJUNCTIVE. */ - public void testOrCorrectnessAndPerf_fuseDualViable() throws Exception { + public void testOrCorrectnessAndPerf() throws Exception { createIndex(); indexDocs(); String ppl = "source = " + INDEX_NAME + " | where match(message, 'hello') or tag = 'hello' | stats count() as cnt"; - Map expected = Map.of( - new MatrixKey(true, false), new ShardStage("lucene", null), - new MatrixKey(true, true), new ShardStage("lucene", null), - new MatrixKey(false, false), new ShardStage("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"), - new MatrixKey(false, true), new ShardStage("datafusion", "CONJUNCTIVE") - ); - runFuseMatrix(ppl, 10L, expected); + try { + // prefer=true: Lucene drives end-to-end, no delegation instruction (no tree_shape). + assertShardStage(ppl, 10L, /* prefer */ true, "lucene", null); + // prefer=false: combiner runs — the dual-viable EQUALS can't stay performance-delegated + // under OR, and it has a correctness sibling (the MATCH), so both collapse into a single + // delegated_predicate — CONJUNCTIVE. + assertShardStage(ppl, 10L, /* prefer */ false, "datafusion", "CONJUNCTIVE"); + } finally { + setPreferMetadataDriver(true); + } } /** - * OR(EQUALS on keyword, EQUALS on integer), oracle = 20. The integer arm isn't - * Lucene-filterable, so the planner picks DataFusion in every cell, and the OR has a - * non-delegatable sibling which keeps the shape INTERLEAVED in both fuse modes. + * OR(EQUALS on keyword, EQUALS on integer), oracle = 20. The keyword EQUALS is dual-viable and, + * under the OR, is reclassified to correctness and shipped to Lucene; the integer EQUALS isn't + * Lucene-filterable and stays native. The two interleave under the OR → INTERLEAVED. */ - public void testOrTwoPerf_fuseDualViable() throws Exception { + public void testOrTwoPerf() throws Exception { createIndex(); indexDocs(); String ppl = "source = " + INDEX_NAME + " | where tag = 'hello' or value = 3 | stats count() as cnt"; - ShardStage interleavedDf = new ShardStage("datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"); - Map expected = Map.of( - new MatrixKey(true, false), interleavedDf, - new MatrixKey(true, true), interleavedDf, - new MatrixKey(false, false), interleavedDf, - new MatrixKey(false, true), interleavedDf - ); - runFuseMatrix(ppl, 20L, expected); - } - - /** Cluster-setting combination: ({@code prefer_metadata_driver}, {@code fuse_dual_viable}). */ - private record MatrixKey(boolean prefer, boolean fuse) {} - - /** Asserted SHARD_FRAGMENT profile fields. {@code treeShape == null} means the field - * must be absent (Lucene-as-driver has no delegation instruction). */ - private record ShardStage(String chosenBackend, String treeShape) {} - - private void runFuseMatrix(String ppl, long oracle, Map expected) throws Exception { try { - for (Map.Entry entry : expected.entrySet()) { - MatrixKey key = entry.getKey(); - ShardStage want = entry.getValue(); - setPreferMetadataDriver(key.prefer()); - setFuseDualViable(key.fuse()); - - String label = "prefer=" + key.prefer() + ",fuse=" + key.fuse(); - assertEquals(label + " — count", oracle, executeCount(ppl)); - Map stage = shardFragmentStage(ppl); - assertEquals(label + " — chosen_backend", want.chosenBackend(), stage.get("chosen_backend")); - assertEquals(label + " — tree_shape", want.treeShape(), stage.get("tree_shape")); - } + // tag (keyword, dual) ships to Lucene under the OR; value (int, native) stays in + // DataFusion → the two interleave under the OR. Same shape both prefer modes. + assertShardStage(ppl, 20L, /* prefer */ true, "datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"); + assertShardStage(ppl, 20L, /* prefer */ false, "datafusion", "INTERLEAVED_BOOLEAN_EXPRESSION"); } finally { - setFuseDualViable(false); setPreferMetadataDriver(true); } } + /** + * Sets {@code prefer_metadata_driver}, then asserts the count oracle and the SHARD_FRAGMENT + * profile's {@code chosen_backend} / {@code tree_shape}. A {@code null} {@code treeShape} means + * the field must be absent (Lucene-as-driver, or no delegation instruction). + */ + private void assertShardStage(String ppl, long oracle, boolean prefer, String chosenBackend, String treeShape) + throws Exception { + setPreferMetadataDriver(prefer); + String label = "prefer=" + prefer; + assertEquals(label + " — count", oracle, executeCount(ppl)); + Map stage = shardFragmentStage(ppl); + assertEquals(label + " — chosen_backend", chosenBackend, stage.get("chosen_backend")); + assertEquals(label + " — tree_shape", treeShape, stage.get("tree_shape")); + } + private long executeCount(String ppl) throws Exception { Map result = executePplViaShim(ppl); @SuppressWarnings("unchecked") @@ -381,12 +372,6 @@ private Map shardFragmentStage(String ppl) throws Exception { throw new AssertionError("No SHARD_FRAGMENT stage in profile: " + stages); } - private void setFuseDualViable(boolean value) throws Exception { - Request req = new Request("PUT", "/_cluster/settings"); - req.setJsonEntity("{\"persistent\":{\"analytics.delegation.fuse_dual_viable\": " + value + "}}"); - client().performRequest(req); - } - private void setPreferMetadataDriver(boolean value) throws Exception { Request req = new Request("PUT", "/_cluster/settings"); req.setJsonEntity("{\"persistent\":{\"analytics.planner.prefer_metadata_driver\": " + value + "}}"); diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/QueryCacheIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/QueryCacheIT.java index 5fbab8d9043d9..c5cd631cae07a 100644 --- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/QueryCacheIT.java +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/QueryCacheIT.java @@ -190,9 +190,9 @@ private void configureCacheSettings(int minFrequency, int costlyMinFrequency) th Request settings = new Request("PUT", "/_cluster/settings"); // Force prefer_metadata_driver=false so the predicate goes through DataFusion's // FilterDelegationHandle / Lucene Collector path — which is what populates the - // query cache. Under prefer=true (default since DELEGATION_FUSE_DUAL_VIABLE flip), - // single-MATCH count fragments take the Lucene-as-driver shortcut via - // IndexSearcher.count, bypassing the cache-tracking collector entirely. + // query cache. Under prefer=true (the default), single-MATCH count fragments take the + // Lucene-as-driver shortcut via IndexSearcher.count, bypassing the cache-tracking + // collector entirely. settings.setJsonEntity("{" + "\"transient\": {" + " \"indices.queries.cache.min_frequency\": " + minFrequency + "," From 540836d861b7c29d8a7aa0da48e914c034709247 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Sat, 6 Jun 2026 22:23:18 +0530 Subject: [PATCH 08/11] Propagate index.sort.field to DataFusion ListingTable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Index-sort settings (`index.sort.field`, `index.sort.order`) are now wired end-to-end into DataFusion's `ListingOptions::with_file_sort_order(...)` so `output_ordering` propagates through `DataSourceExec` and downstream optimizers can fire (`sort_prefix` on TopK, `split_file_groups_by_statistics`, single-file groups when target_partitions ≥ num_files). Java: ReaderManagerConfig carries IndexSettings; DataFusionPlugin reads the sort settings from IndexSettings and threads them via DatafusionReaderManager → DatafusionReader → ReaderHandle → NativeBridge.createDatafusionReader as parallel String[] arrays plus a count. Rust: ffm.rs decodes the new FFM args; api.rs / ShardView store them; the session-context build site builds a Vec using `Expr::Column(Column::from_name(...))` (case-preserving — `col(&str)` would silently lowercase the column name) and calls `with_file_sort_order(...)`. Sets `split_file_groups_by_statistics = true` and matches `target_partitions` to `effective_partitions` when sort fields are present. Also includes a small fix to the IndexedTable provider's table-provider construction (multi-segment scan path) and a Spotless-driven whitespace fix in ToStringFunctionAdapter that came along with the patch. Tests: IndexSortPropagationIT (internal cluster); existing tests updated to pass the two new List args at every call site. Signed-off-by: Arpit Bandejiya --- .../rust/src/api.rs | 23 ++ .../rust/src/ffm.rs | 36 ++- .../rust/src/query_executor.rs | 27 +- .../rust/src/session_context.rs | 46 ++- .../be/datafusion/DataFusionPlugin.java | 30 +- .../be/datafusion/DatafusionReader.java | 12 +- .../datafusion/DatafusionReaderManager.java | 21 +- .../be/datafusion/nativelib/NativeBridge.java | 36 ++- .../be/datafusion/nativelib/ReaderHandle.java | 12 +- .../DataFusionNativeBridgeTests.java | 12 +- .../DataFusionQueryExecutionTests.java | 8 +- .../DatafusionReaderManagerTests.java | 24 +- .../DatafusionResultStreamTests.java | 8 +- .../DatafusionSearchExecEngineTests.java | 8 +- .../be/lucene/LuceneReaderManagerTests.java | 6 +- .../CompositeIndexingExecutionEngine.java | 3 +- .../be/datafusion/IndexSortPropagationIT.java | 273 ++++++++++++++++++ .../index/engine/DataFormatAwareEngine.java | 3 +- .../DataFormatAwareNRTReplicationEngine.java | 2 +- .../engine/DataFormatAwareReadOnlyEngine.java | 3 +- .../dataformat/ReaderManagerConfig.java | 5 +- .../dataformat/DataFormatRegistryTests.java | 2 +- 22 files changed, 559 insertions(+), 41 deletions(-) create mode 100644 sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/IndexSortPropagationIT.java diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index 0bf126101aa6c..766f57fe22797 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -330,6 +330,16 @@ pub struct ShardView { /// this routes reads through TieredObjectStore (local + remote). /// When no store is provided, uses default LocalFileSystem. pub store: Arc, + /// Index sort fields, in priority order. Sourced from the index's + /// `index.sort.field` setting on the Java side. Empty when the index has + /// no `index.sort.field` configured. Parallel to `sort_orders`. + /// Used to build `ListingOptions.with_file_sort_order(...)` so DataFusion + /// advertises `output_ordering` from the scan and enables the + /// `sort_prefix` optimization on TopK / SortPreservingMerge. + pub sort_fields: Vec, + /// Index sort directions per field — values: `"asc"` or `"desc"`. + /// Parallel to `sort_fields`. Sourced from `index.sort.order`. + pub sort_orders: Vec, } /// Creates a DataFusion global runtime with the given resource limits. @@ -500,6 +510,8 @@ pub fn create_reader( table_path: &str, filenames: Vec, writer_generations: Vec, + sort_fields: Vec, + sort_orders: Vec, tokio_rt_manager: &RuntimeManager, store_ptr: i64, ) -> Result { @@ -510,6 +522,13 @@ pub fn create_reader( writer_generations.len() ))); } + if sort_fields.len() != sort_orders.len() { + return Err(DataFusionError::Execution(format!( + "create_reader: sort_fields ({}) and sort_orders ({}) must have the same length", + sort_fields.len(), + sort_orders.len() + ))); + } let table_url = ListingTableUrl::parse(table_path) .map_err(|e| DataFusionError::Execution(format!("Invalid table path: {}", e)))?; @@ -536,6 +555,8 @@ pub fn create_reader( writer_generations: Arc::new(writer_generations), file_metadata: None, store, + sort_fields, + sort_orders, }; Ok(Box::into_raw(Box::new(shard_view)) as i64) } @@ -666,6 +687,8 @@ pub async unsafe fn execute_query( context_id, Arc::clone(&shard_view.store), phantom_corrector, + &shard_view.sort_fields, + &shard_view.sort_orders, ).await } }; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index fc201d1397176..2c8fcc22c6fe8 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -189,6 +189,11 @@ pub unsafe extern "C" fn df_create_reader( writer_generations_ptr: *const i64, files_count: i64, store_ptr: i64, + sort_fields_ptr: *const *const u8, + sort_fields_len_ptr: *const i64, + sort_orders_ptr: *const *const u8, + sort_orders_len_ptr: *const i64, + sort_count: i64, ) -> i64 { let table_path = str_from_raw(table_path_ptr, table_path_len) .map_err(|e| format!("df_create_reader: {}", e))?; @@ -204,8 +209,37 @@ pub unsafe extern "C" fn df_create_reader( ); writer_generations.push(*writer_generations_ptr.add(i)); } + // Decode parallel sort_fields / sort_orders String arrays. sort_count == 0 means no + // index sort configured; pass an empty Vec. + let mut sort_fields = Vec::with_capacity(sort_count as usize); + let mut sort_orders = Vec::with_capacity(sort_count as usize); + for i in 0..sort_count as usize { + let f_ptr = *sort_fields_ptr.add(i); + let f_len = *sort_fields_len_ptr.add(i); + sort_fields.push( + str_from_raw(f_ptr, f_len) + .map_err(|e| format!("df_create_reader: sort_field[{}]: {}", i, e))? + .to_string(), + ); + let o_ptr = *sort_orders_ptr.add(i); + let o_len = *sort_orders_len_ptr.add(i); + sort_orders.push( + str_from_raw(o_ptr, o_len) + .map_err(|e| format!("df_create_reader: sort_order[{}]: {}", i, e))? + .to_string(), + ); + } let mgr = get_rt_manager()?; - api::create_reader(table_path, filenames, writer_generations, &mgr, store_ptr).map_err(|e| e.to_string()) + api::create_reader( + table_path, + filenames, + writer_generations, + sort_fields, + sort_orders, + &mgr, + store_ptr, + ) + .map_err(|e| e.to_string()) } #[no_mangle] diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index e695c7d2bed23..5e242dd2d44ad 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -53,6 +53,8 @@ pub async fn execute_query( context_id: i64, shard_store: Arc, phantom_corrector: Option>, + sort_fields: &[String], + sort_orders: &[String], ) -> Result { // Build per-query RuntimeEnv with list-files cache pre-populated. let runtime_env = build_query_runtime_env(runtime, &table_path, object_metas.as_ref())?; @@ -135,9 +137,32 @@ pub async fn execute_query( _ => { // Baseline: use standard ListingTable let file_format = ParquetFormat::new(); - let listing_options = ListingOptions::new(Arc::new(file_format)) + let mut listing_options = ListingOptions::new(Arc::new(file_format)) .with_file_extension(".parquet") .with_collect_stat(true); + // Declare per-file sort order to DataFusion if the index has `index.sort.field`. + // OpenSearch's parquet writer enforces this sort within each segment, so the claim + // is verifiable-by-construction. With `output_ordering` advertised on + // `DataSourceExec`, the optimizer can elide redundant SortExec, enable + // `repartition_preserving_order` (one whole file per partition), and fire the + // `sort_prefix` TopK optimization when the query's ORDER BY matches a prefix. + if !sort_fields.is_empty() { + use datafusion::common::Column; + use datafusion::logical_expr::{Expr, SortExpr}; + let sort_exprs: Vec = sort_fields + .iter() + .zip(sort_orders.iter()) + .map(|(name, order)| { + let asc = order.eq_ignore_ascii_case("asc"); + // Match Lucene default NULLS placement: ASC → first, DESC → last. + // Use `Column::from_name` (NOT `col(name)`) to preserve the column-name + // case. `col("EventTime")` lowercases via SQL identifier normalization, + // breaking lookups against Arrow schemas with PascalCase names. + Expr::Column(Column::from_name(name.clone())).sort(asc, asc) + }) + .collect(); + listing_options = listing_options.with_file_sort_order(vec![sort_exprs]); + } let resolved_schema = listing_options .infer_schema(&ctx.state(), &table_path) .await diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs index e1d5ff8eead49..50e074c087923 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs @@ -13,6 +13,7 @@ use std::sync::Arc; +use native_bridge_common::log_debug; use datafusion::{ common::DataFusionError, datasource::file_format::parquet::ParquetFormat, @@ -204,6 +205,11 @@ pub async unsafe fn create_session_context( config.options_mut().execution.parquet.pushdown_filters = query_config.parquet_pushdown_filters; config.options_mut().execution.target_partitions = effective_partitions; config.options_mut().execution.batch_size = effective_batch_size; + // When the index has `index.sort.field`, ask DataFusion to use the sort-aware + // file-group partitioner so `output_ordering` can propagate from the scan. + if !shard_view.sort_fields.is_empty() { + config.options_mut().execution.split_file_groups_by_statistics = true; + } let mut state_builder = SessionStateBuilder::new() .with_config(config) @@ -237,9 +243,40 @@ pub async unsafe fn create_session_context( crate::udwf::register_all(&ctx); // Register default ListingTable for parquet scans. - let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default())) + // + // `target_partitions` on the listing options drives the sort-aware bin-packer in + // `split_groups_by_statistics_with_target_partitions`. We set it to the session's + // effective partition count so the bin-packer produces up to N groups (one file per + // group when min/max ranges can't chain). The session-state's `target_partitions` + // controls EnforceDistribution; this one is independent. + let mut listing_options = ListingOptions::new(Arc::new(ParquetFormat::default())) .with_file_extension(".parquet") - .with_collect_stat(true); + .with_collect_stat(true) + .with_target_partitions(effective_partitions); + + // When the index has `index.sort.field`, declare the per-file sort order so + // DataFusion can advertise `output_ordering` through the scan and enable the + // `sort_prefix` TopK optimization. The OpenSearch writer enforces this sort + // within each segment, so the per-file claim is verifiable-by-construction. + // + // `Column::from_name` (instead of `col(name)`) preserves the column-name case. + // `col("EventTime")` lowercases via SQL identifier normalization to "eventtime", + // which fails the lookup against Arrow schemas that store the case-sensitive name. + if !shard_view.sort_fields.is_empty() { + use datafusion::common::Column; + use datafusion::logical_expr::{Expr, SortExpr}; + let sort_exprs: Vec = shard_view + .sort_fields + .iter() + .zip(shard_view.sort_orders.iter()) + .map(|(name, order)| { + let asc = order.eq_ignore_ascii_case("asc"); + // Match Lucene default NULLS placement: ASC → NULLS FIRST, DESC → NULLS LAST. + Expr::Column(Column::from_name(name.clone())).sort(asc, asc) + }) + .collect(); + listing_options = listing_options.with_file_sort_order(vec![sort_exprs]); + } // For multi-index queries, the plan's NamedTable carries the logical name (alias/pattern) // which differs from table_name (the concrete shard index). Extract it from the plan and @@ -314,6 +351,11 @@ pub async unsafe fn create_session_context( ); e })?; + log_debug!( + "create_session_context: registered table '{}' with file_sort_order_keys={}", + register_name, + shard_view.sort_fields.len() + ); error!( "create_session_context: successfully registered table '{}', table_name_len={}", diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index 78baa761dbe56..14d720ed77e87 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -33,9 +33,12 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.IndexSortConfig; import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.ReaderManagerConfig; import org.opensearch.index.engine.exec.EngineReaderManager; +import org.opensearch.search.sort.SortOrder; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.monitor.os.OsProbe; import org.opensearch.nativebridge.spi.NativeMemoryFetcher; @@ -615,7 +618,32 @@ public String name() { @Override public EngineReaderManager createReaderManager(ReaderManagerConfig settings) throws IOException { NativeStoreHandle dataformatAwareStoreHandle = settings.dataformatAwareStoreHandles().get(settings.format()); - return new DatafusionReaderManager(settings.format(), settings.shardPath(), dataFusionService, dataformatAwareStoreHandle); + // Pull index.sort.field / index.sort.order off IndexSettings so the native reader can declare + // file sort order to DataFusion. Empty lists when the index has no index sort configured. + List sortFields = List.of(); + List sortOrders = List.of(); + IndexSettings indexSettings = settings.indexSettings(); + if (indexSettings != null) { + List fields = IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexSettings.getSettings()); + List orders = IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(indexSettings.getSettings()); + if (fields != null && !fields.isEmpty()) { + sortFields = List.copyOf(fields); + if (orders != null && orders.size() == fields.size()) { + sortOrders = orders.stream().map(o -> o == SortOrder.DESC ? "desc" : "asc").toList(); + } else { + // index.sort.order defaults to asc per field when omitted + sortOrders = fields.stream().map(f -> "asc").toList(); + } + } + } + return new DatafusionReaderManager( + settings.format(), + settings.shardPath(), + dataFusionService, + dataformatAwareStoreHandle, + sortFields, + sortOrders + ); } @Override diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReader.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReader.java index a1777077130ea..2d24befd3a9e2 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReader.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReader.java @@ -47,8 +47,16 @@ public class DatafusionReader implements Closeable { * @param directoryPath shard data directory * @param writerFileSets the per-segment file sets from the catalog snapshot * @param dataformatAwareStoreHandle per-format native store handle (null on hot, live on warm) + * @param sortFields index.sort.field values (empty list when the index has no sort). Parallel to {@code sortOrders}. + * @param sortOrders index.sort.order values ("asc"/"desc"), parallel to {@code sortFields}. */ - public DatafusionReader(String directoryPath, Collection writerFileSets, NativeStoreHandle dataformatAwareStoreHandle) { + public DatafusionReader( + String directoryPath, + Collection writerFileSets, + NativeStoreHandle dataformatAwareStoreHandle, + List sortFields, + List sortOrders + ) { this.directoryPath = directoryPath; List segments; if (writerFileSets == null || writerFileSets.isEmpty()) { @@ -56,7 +64,7 @@ public DatafusionReader(String directoryPath, Collection writerFi } else { segments = writerFileSets.stream().map(MonoFileWriterSet::from).toList(); } - readerHandle = new ReaderHandle(directoryPath, segments, dataformatAwareStoreHandle); + readerHandle = new ReaderHandle(directoryPath, segments, dataformatAwareStoreHandle, sortFields, sortOrders); } /** diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReaderManager.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReaderManager.java index 51943256c8a38..ab437bd3a164a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReaderManager.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReaderManager.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -42,6 +43,14 @@ public class DatafusionReaderManager implements EngineReaderManager sortFields; + /** Parallel to {@link #sortFields}; values are {@code "asc"} or {@code "desc"}. */ + private final List sortOrders; /** * Creates a reader manager. @@ -51,17 +60,23 @@ public class DatafusionReaderManager implements EngineReaderManager sortFields, + List sortOrders ) { this.dataFormat = dataFormat; this.directoryPath = shardPath.getDataPath().resolve(dataFormat.name()).toString(); this.dataFusionService = dataFusionService; this.dataformatAwareStoreHandle = dataformatAwareStoreHandle; + this.sortFields = sortFields == null ? List.of() : sortFields; + this.sortOrders = sortOrders == null ? List.of() : sortOrders; } @Override @@ -106,7 +121,9 @@ public void afterRefresh(boolean didRefresh, CatalogSnapshot catalogSnapshot) th DatafusionReader reader = new DatafusionReader( directoryPath, catalogSnapshot.getSearchableFiles(dataFormat.name()), - dataformatAwareStoreHandle + dataformatAwareStoreHandle, + sortFields, + sortOrders ); readers.put(catalogSnapshot.getId(), reader); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 5d7604edbf1a1..e3b224f9406f9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -208,8 +208,13 @@ public final class NativeBridge { ValueLayout.ADDRESS, // files_ptr ValueLayout.ADDRESS, // files_len_ptr ValueLayout.ADDRESS, // writer_generations_ptr - ValueLayout.JAVA_LONG, // count (applies to all three parallel arrays) - ValueLayout.JAVA_LONG // object store ptr + ValueLayout.JAVA_LONG, // count (applies to filenames + writer_generations) + ValueLayout.JAVA_LONG, // object store ptr + ValueLayout.ADDRESS, // sort_fields_ptr (parallel String[] for index.sort.field) + ValueLayout.ADDRESS, // sort_fields_len_ptr + ValueLayout.ADDRESS, // sort_orders_ptr (parallel String[] for index.sort.order: "asc"|"desc") + ValueLayout.ADDRESS, // sort_orders_len_ptr + ValueLayout.JAVA_LONG // sort_count (0 when index has no sort) ) ); @@ -786,11 +791,18 @@ public static void setMemoryGuardThresholds( * @param path shard data directory * @param segments per-segment metadata — each carries a single filename and writer generation * @param dataformatAwareStoreHandle per-format native store handle (null = local, live = use store pointer) + * @param sortFields index.sort.field values, or empty list if the index has no sort configured. + * Parallel to {@code sortOrders}. + * @param sortOrders index.sort.order values ("asc" or "desc"), parallel to {@code sortFields}. + * Used by Rust to call {@code ListingOptions.with_file_sort_order(...)} so the + * parquet scan advertises {@code output_ordering} to the DataFusion optimizer. */ public static long createDatafusionReader( String path, List segments, - NativeStoreHandle dataformatAwareStoreHandle + NativeStoreHandle dataformatAwareStoreHandle, + List sortFields, + List sortOrders ) { long storePtr = 0L; if (dataformatAwareStoreHandle != null) { @@ -801,13 +813,29 @@ public static long createDatafusionReader( storePtr = 0L; } } + if (sortFields == null) sortFields = List.of(); + if (sortOrders == null) sortOrders = List.of(); + if (sortFields.size() != sortOrders.size()) { + throw new IllegalArgumentException( + "createDatafusionReader: sortFields (" + sortFields.size() + + ") and sortOrders (" + sortOrders.size() + ") must have the same length" + ); + } try (var call = new NativeCall()) { var p = call.str(path); var f = call.strArray(segments.stream().map(org.opensearch.index.engine.exec.MonoFileWriterSet::file).toArray(String[]::new)); var gens = call.longs( segments.stream().mapToLong(org.opensearch.index.engine.exec.MonoFileWriterSet::writerGeneration).toArray() ); - return call.invoke(CREATE_READER, p.segment(), p.len(), f.ptrs(), f.lens(), gens, f.count(), storePtr); + var sf = call.strArray(sortFields.toArray(String[]::new)); + var so = call.strArray(sortOrders.toArray(String[]::new)); + return call.invoke( + CREATE_READER, + p.segment(), p.len(), + f.ptrs(), f.lens(), gens, f.count(), + storePtr, + sf.ptrs(), sf.lens(), so.ptrs(), so.lens(), sf.count() + ); } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/ReaderHandle.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/ReaderHandle.java index 57a6576350d03..101f0a6926f2c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/ReaderHandle.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/ReaderHandle.java @@ -29,9 +29,17 @@ public final class ReaderHandle extends NativeHandle { * @param path the directory path containing data files * @param segments the per-segment file sets to read * @param dataformatAwareStoreHandle per-format native store handle (null = local, live = use store pointer) + * @param sortFields index.sort.field values (or empty if no index sort). Parallel to {@code sortOrders}. + * @param sortOrders index.sort.order values ("asc"/"desc"), parallel to {@code sortFields}. */ - public ReaderHandle(String path, List segments, NativeStoreHandle dataformatAwareStoreHandle) { - super(NativeBridge.createDatafusionReader(path, segments, dataformatAwareStoreHandle)); + public ReaderHandle( + String path, + List segments, + NativeStoreHandle dataformatAwareStoreHandle, + List sortFields, + List sortOrders + ) { + super(NativeBridge.createDatafusionReader(path, segments, dataformatAwareStoreHandle, sortFields, sortOrders)); this.ownsPointer = true; } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java index bef5c04af4d63..6f87b6c85ccde 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java @@ -72,7 +72,9 @@ public void testReaderLifecycle() throws Exception { ReaderHandle readerHandle = new ReaderHandle( dataDir.toString(), java.util.List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), - null + null, + java.util.List.of(), + java.util.List.of() ); assertTrue("Reader pointer should be non-zero", readerHandle.getPointer() != 0); @@ -95,7 +97,9 @@ public void testSessionContextCreationAndTableRegistration() throws Exception { ReaderHandle readerHandle = new ReaderHandle( dataDir.toString(), java.util.List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), - null + null, + java.util.List.of(), + java.util.List.of() ); // Create session context with table registered long queryConfigPtr; @@ -184,7 +188,9 @@ public void testReaderWithRealNativeStoreHandle() throws Exception { ReaderHandle readerHandle = new ReaderHandle( dataDir.toString(), List.of(MonoFileWriterSet.of(dataDir.toString(), 1L, "test.parquet", 0L)), - storeHandle + storeHandle, + List.of(), + List.of() ); assertTrue("Reader pointer should be non-zero", readerHandle.getPointer() != 0); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java index 617fbe3e995ba..307d0e5e15421 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java @@ -64,7 +64,13 @@ public void setUp() throws Exception { Path dataDir = createTempDir("datafusion-data"); Path testParquet = Path.of(getClass().getClassLoader().getResource("test.parquet").toURI()); Files.copy(testParquet, dataDir.resolve("test.parquet")); - readerHandle = new ReaderHandle(dataDir.toString(), List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), storeHandle); + readerHandle = new ReaderHandle( + dataDir.toString(), + List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), + storeHandle, + List.of(), + List.of() + ); configArena = Arena.ofConfined(); MemorySegment configSegment = configArena.allocate(WireConfigSnapshot.BYTE_SIZE); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReaderManagerTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReaderManagerTests.java index e583f832aee4d..1acb205cad32f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReaderManagerTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReaderManagerTests.java @@ -71,7 +71,7 @@ public void testConstructorAcceptsNullHandle() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); assertNotNull(manager); manager.close(); } @@ -83,7 +83,7 @@ public void testConstructorAcceptsEmptyHandle() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, NativeStoreHandle.EMPTY); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, NativeStoreHandle.EMPTY, List.of(), List.of()); assertNotNull(manager); manager.close(); } @@ -95,7 +95,7 @@ public void testCloseWithNoReadersDoesNotThrow() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); // Should not throw — no readers to close manager.close(); } @@ -107,7 +107,7 @@ public void testOnFilesDeletedWithNullIsNoOp() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); manager.onFilesDeleted(null); verifyNoInteractions(mockService); manager.close(); @@ -120,7 +120,7 @@ public void testOnFilesDeletedWithEmptyCollectionIsNoOp() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); manager.onFilesDeleted(List.of()); verifyNoInteractions(mockService); manager.close(); @@ -133,7 +133,7 @@ public void testOnFilesAddedWithNullIsNoOp() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); manager.onFilesAdded(null); verifyNoInteractions(mockService); manager.close(); @@ -146,7 +146,7 @@ public void testOnFilesAddedWithEmptyCollectionIsNoOp() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); manager.onFilesAdded(List.of()); verifyNoInteractions(mockService); manager.close(); @@ -159,7 +159,7 @@ public void testOnFilesDeletedDelegatesToService() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); Collection files = List.of("seg_0.parquet", "seg_1.parquet"); manager.onFilesDeleted(files); @@ -176,7 +176,7 @@ public void testOnFilesAddedDelegatesToService() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); Collection files = List.of("seg_0.parquet"); manager.onFilesAdded(files); @@ -193,7 +193,7 @@ public void testBeforeRefreshDoesNotThrow() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); manager.beforeRefresh(); manager.close(); } @@ -205,7 +205,7 @@ public void testAfterRefreshWithDidRefreshFalseIsNoOp() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); // didRefresh=false means no new data — should be a no-op manager.afterRefresh(false, null); manager.close(); @@ -218,7 +218,7 @@ public void testGetReaderWithNoRefreshThrows() throws IOException { ShardPath shardPath = createTestShardPath(); DataFusionService mockService = mock(DataFusionService.class); - DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null); + DatafusionReaderManager manager = new DatafusionReaderManager(TEST_FORMAT, shardPath, mockService, null, List.of(), List.of()); expectThrows(IllegalArgumentException.class, () -> manager.getReader(null)); manager.close(); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java index 9bb7c44746939..28db10793bffc 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java @@ -59,7 +59,13 @@ public void setUp() throws Exception { Path dataDir = createTempDir("data"); Path testParquet = Path.of(getClass().getClassLoader().getResource("test.parquet").toURI()); Files.copy(testParquet, dataDir.resolve("test.parquet")); - readerHandle = new ReaderHandle(dataDir.toString(), List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), storeHandle); + readerHandle = new ReaderHandle( + dataDir.toString(), + List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), + storeHandle, + List.of(), + List.of() + ); configArena = Arena.ofConfined(); MemorySegment configSegment = configArena.allocate(WireConfigSnapshot.BYTE_SIZE); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java index 5d5b4c7ff749c..438c5c091a74d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java @@ -55,7 +55,13 @@ public void setUp() throws Exception { Path dataDir = createTempDir("datafusion-data"); Path testParquet = Path.of(getClass().getClassLoader().getResource("test.parquet").toURI()); Files.copy(testParquet, dataDir.resolve("test.parquet")); - readerHandle = new ReaderHandle(dataDir.toString(), List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), storeHandle); + readerHandle = new ReaderHandle( + dataDir.toString(), + List.of(MonoFileWriterSet.of(".", 0L, "test.parquet", 0L)), + storeHandle, + List.of(), + List.of() + ); } @Override diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java index e8afb62e8f7ad..df2dcde0cbe65 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java @@ -671,7 +671,8 @@ public void testCreateReaderManagerWithLuceneIndexingEngine() throws IOException dataFormat, mock(DataFormatRegistry.class), shardPath, - Map.of() + Map.of(), + null ); EngineReaderManager rm = LuceneSearchBackEnd.createReaderManager(settings); @@ -688,7 +689,8 @@ public void testCreateReaderManagerWithEmptyProviderThrows() { dataFormat, mock(DataFormatRegistry.class), null, - Map.of() + Map.of(), + null ); IllegalStateException ex = expectThrows(IllegalStateException.class, () -> LuceneSearchBackEnd.createReaderManager(settings)); diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index 95bf52af4d98f..47ece1f7d36ea 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -542,7 +542,8 @@ private ReaderManagerConfig readerManagerConfig(ReaderManagerConfig config, Data toAugment, config.registry(), config.shardPath(), - config.dataformatAwareStoreHandles() + config.dataformatAwareStoreHandles(), + config.indexSettings() ); } diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/IndexSortPropagationIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/IndexSortPropagationIT.java new file mode 100644 index 0000000000000..32bc72ea53c68 --- /dev/null +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/be/datafusion/IndexSortPropagationIT.java @@ -0,0 +1,273 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.analytics.AnalyticsPlugin; +import org.opensearch.analytics.exec.DefaultPlanExecutor; +import org.opensearch.analytics.sql.SqlPlanRunner; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.arrow.flight.transport.FlightStreamPlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +/** + * Correctness IT for the {@code index.sort.field} → DataFusion {@code with_file_sort_order(...)} + * propagation in the analytics-backend-datafusion vanilla query path. + * + *

    Background. The DatafusionPlugin reads {@code index.sort.field} / {@code index.sort.order} off + * {@link org.opensearch.index.IndexSettings}, ships them across FFM in {@code df_create_reader}, + * and threads them into {@link org.opensearch.index.engine.exec.EngineReaderManager} → + * {@code ShardView.sort_fields}/{@code sort_orders}. At session-context build time + * (Rust-side {@code session_context.rs}) the lists become a + * {@code ListingOptions::with_file_sort_order(...)} call, which makes DataFusion advertise + * {@code output_ordering} from {@code DataSourceExec} and unlock the + * {@code repartition_preserving_order} / {@code sort_prefix} optimizations. + * + *

    {@code with_file_sort_order} is a per-file claim — DataFusion does not verify it. The + * claim is safe here only because the OpenSearch parquet writer enforces the same sort within + * each segment via {@code index.sort.field}. The risk we're guarding against in this IT is a + * regression where: + * + *

      + *
    • the claim doesn't match the writer's actual physical layout, or
      • + *
    • the optimizer mis-uses the claim and silently drops or re-orders rows.
      • + *
    + * + *

    Strategy: ingest the same set of rows into two indices — one with {@code index.sort.field} + * configured, one without — and assert that the query results are byte-identical for several + * query shapes (filter-only, sort+limit, agg+sort). If our patch lies, results would differ. + * + * @opensearch.internal + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2, numClientNodes = 0) +public class IndexSortPropagationIT extends OpenSearchIntegTestCase { + + private static final String IDX_UNSORTED = "sort_prop_unsorted"; + private static final String IDX_SORTED = "sort_prop_sorted"; + private static final int DOC_COUNT = 30; + private static final int SHARDS = 2; + + @Override + protected Collection> nodePlugins() { + return List.of(ArrowBasePlugin.class, CompositeDataFormatPlugin.class, MockCommitterEnginePlugin.class); + } + + @Override + protected Collection additionalNodePlugins() { + return List.of( + classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), + classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) + ); + } + + private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) { + return new PluginInfo( + pluginClass.getName(), + "classpath plugin", + "NA", + Version.CURRENT, + "1.8", + pluginClass.getName(), + null, + extendedPlugins, + false + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(FeatureFlags.STREAM_TRANSPORT, true) + .build(); + } + + /** + * Filter-only query — no sort, no limit. Must produce the same rows in both indices. + */ + public void testFilterOnly_resultsMatchAcrossSortedAndUnsorted() throws Exception { + createAndSeedBothIndices(); + SqlPlanRunner runner = sqlPlanRunner(); + + List unsorted = runner.executeSql( + "SELECT URL, EventDate, CounterID FROM " + IDX_UNSORTED + " WHERE CounterID > 5" + ); + List sorted = runner.executeSql( + "SELECT URL, EventDate, CounterID FROM " + IDX_SORTED + " WHERE CounterID > 5" + ); + + assertResultsEquivalent("filter-only", unsorted, sorted); + } + + /** + * Sort+LIMIT — the query that actually exercises {@code sort_prefix} when the ORDER BY + * matches a prefix of {@code index.sort.field}. Both indices must produce the same top-N + * rows in the same order. + */ + public void testSortLimit_resultsMatchAcrossSortedAndUnsorted() throws Exception { + createAndSeedBothIndices(); + SqlPlanRunner runner = sqlPlanRunner(); + + // ORDER BY (CounterID DESC, EventDate DESC) is a prefix of the sort declared on IDX_SORTED. + String sql = " WHERE CounterID > 0 ORDER BY CounterID DESC, EventDate DESC LIMIT 10"; + List unsorted = runner.executeSql("SELECT URL, EventDate, CounterID FROM " + IDX_UNSORTED + sql); + List sorted = runner.executeSql("SELECT URL, EventDate, CounterID FROM " + IDX_SORTED + sql); + + assertEquals("LIMIT 10 must produce 10 rows for unsorted", 10, unsorted.size()); + assertEquals("LIMIT 10 must produce 10 rows for sorted", 10, sorted.size()); + // For sort+LIMIT, row ORDER must match — not just multiset equality. + assertOrderedRowsEqual("sort+limit", unsorted, sorted); + } + + /** + * Agg + sort on aggregate output — the sort declaration must NOT bleed into queries that + * don't reference the sort columns in their ORDER BY. + */ + public void testAggSortOnAggregate_resultsMatchAcrossSortedAndUnsorted() throws Exception { + createAndSeedBothIndices(); + SqlPlanRunner runner = sqlPlanRunner(); + + String sql = " WHERE CounterID > 0 GROUP BY EventDate ORDER BY COUNT(*) DESC, EventDate ASC LIMIT 5"; + List unsorted = runner.executeSql("SELECT EventDate, COUNT(*) AS c FROM " + IDX_UNSORTED + sql); + List sorted = runner.executeSql("SELECT EventDate, COUNT(*) AS c FROM " + IDX_SORTED + sql); + + // tiebreaker-stable ORDER BY (count, then date) → ordered comparison is meaningful. + assertOrderedRowsEqual("agg+sort", unsorted, sorted); + } + + /** + * ORDER BY a non-sort column. The sort declaration must not re-route results based on + * unrelated columns. + */ + public void testSortOnNonSortColumn_resultsMatchAcrossSortedAndUnsorted() throws Exception { + createAndSeedBothIndices(); + SqlPlanRunner runner = sqlPlanRunner(); + + String sql = " WHERE CounterID > 0 ORDER BY URL ASC LIMIT 8"; + List unsorted = runner.executeSql("SELECT URL, CounterID FROM " + IDX_UNSORTED + sql); + List sorted = runner.executeSql("SELECT URL, CounterID FROM " + IDX_SORTED + sql); + + assertOrderedRowsEqual("sort-on-non-sort-column", unsorted, sorted); + } + + // ── Infrastructure ────────────────────────────────────────────────────── + + private SqlPlanRunner sqlPlanRunner() { + String node = internalCluster().getNodeNames()[0]; + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node); + DefaultPlanExecutor executor = internalCluster().getInstance(DefaultPlanExecutor.class, node); + return new SqlPlanRunner(clusterService, executor); + } + + /** + * Creates two parallel indices ingested with the same docs: + * - {@code IDX_UNSORTED}: no {@code index.sort.field}. + * - {@code IDX_SORTED}: {@code index.sort.field=[CounterID, EventDate]} both descending. + */ + private void createAndSeedBothIndices() { + createIndex(IDX_UNSORTED, /*sortFields*/ null, /*sortOrders*/ null); + createIndex(IDX_SORTED, List.of("CounterID", "EventDate"), List.of("desc", "desc")); + seed(IDX_UNSORTED); + seed(IDX_SORTED); + } + + private void createIndex(String name, List sortFields, List sortOrders) { + Settings.Builder settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats"); + if (sortFields != null && !sortFields.isEmpty()) { + settings.putList("index.sort.field", sortFields); + settings.putList("index.sort.order", sortOrders); + } + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(name) + .setSettings(settings.build()) + .setMapping("URL", "type=keyword", "EventDate", "type=date", "CounterID", "type=integer") + .get(); + assertTrue("index creation must be acknowledged for " + name, response.isAcknowledged()); + ensureGreen(name); + } + + /** + * Seeds the same {@code DOC_COUNT} docs into the given index. Doc IDs are stable so both + * indices end up with identical content. + */ + private void seed(String index) { + for (int i = 0; i < DOC_COUNT; i++) { + // CounterID: 1..DOC_COUNT, EventDate spans 2026-05-01..2026-05-(1 + i mod 10). + // Same content for both indices — only physical layout differs because of index.sort.field. + client().prepareIndex(index) + .setId(String.valueOf(i)) + .setSource( + "URL", + "https://example.com/page" + i, + "EventDate", + "2026-05-" + String.format(Locale.ROOT, "%02d", (i % 10) + 1), + "CounterID", + i + 1 + ) + .get(); + } + client().admin().indices().prepareRefresh(index).get(); + client().admin().indices().prepareFlush(index).get(); + } + + /** + * Multiset comparison — for queries where row order isn't guaranteed (e.g. filter-only). + * Sorts both lists by a stable key derived from each row before comparing. + */ + private static void assertResultsEquivalent(String label, List a, List b) { + assertEquals(label + ": row count mismatch", a.size(), b.size()); + List aKeys = a.stream().map(IndexSortPropagationIT::rowKey).sorted().toList(); + List bKeys = b.stream().map(IndexSortPropagationIT::rowKey).sorted().toList(); + assertEquals(label + ": row content mismatch", aKeys, bKeys); + } + + /** + * Order-sensitive comparison — for queries with a deterministic ORDER BY. + */ + private static void assertOrderedRowsEqual(String label, List a, List b) { + assertEquals(label + ": row count mismatch", a.size(), b.size()); + for (int i = 0; i < a.size(); i++) { + assertEquals(label + ": row " + i + " mismatch", rowKey(a.get(i)), rowKey(b.get(i))); + } + } + + private static String rowKey(Object[] row) { + StringBuilder sb = new StringBuilder(); + for (Object cell : row) { + sb.append(cell == null ? "" : cell.toString()).append('|'); + } + return sb.toString(); + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java index 74147fd3777f0..3e7dbcf429dbb 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java @@ -347,7 +347,8 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { indexingExecutionEngine.getDataFormat(), registry, store.shardPath(), - store.getDataformatAwareStoreHandles() + store.getDataformatAwareStoreHandles(), + engineConfig.getIndexSettings() ) ); diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareNRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareNRTReplicationEngine.java index 0a1a49b1ea90d..f8fcd59cf723d 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareNRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareNRTReplicationEngine.java @@ -174,7 +174,7 @@ public Store store() { } }; } - }), format, registry, store.shardPath(), store.getDataformatAwareStoreHandles()))); + }), format, registry, store.shardPath(), store.getDataformatAwareStoreHandles(), engineConfig.getIndexSettings()))); } readerManagersRef = Map.copyOf(aggregated); diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareReadOnlyEngine.java index 5beb7144625f3..0817a968cd1fb 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareReadOnlyEngine.java @@ -159,7 +159,8 @@ public DataFormatAwareReadOnlyEngine(EngineConfig engineConfig) { format, registry, store.shardPath(), - store.getDataformatAwareStoreHandles() + store.getDataformatAwareStoreHandles(), + engineConfig.getIndexSettings() ) ) ); diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/ReaderManagerConfig.java b/server/src/main/java/org/opensearch/index/engine/dataformat/ReaderManagerConfig.java index 204cac4b7b02e..675112e71d965 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/ReaderManagerConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/ReaderManagerConfig.java @@ -9,6 +9,7 @@ package org.opensearch.index.engine.dataformat; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.exec.commit.IndexStoreProvider; import org.opensearch.index.shard.ShardPath; import org.opensearch.plugins.NativeStoreHandle; @@ -28,10 +29,12 @@ * @param dataformatAwareStoreHandles per-format native store handles for reads. * Empty map if no native stores are available. * Each plugin extracts its own handle via {@code handles.get(config.format())}. + * @param indexSettings the index settings (carries {@code IndexSortConfig} so backends can declare + * file sort order to their query optimizers). * * @opensearch.experimental */ @ExperimentalApi public record ReaderManagerConfig(Optional indexStoreProvider, DataFormat format, DataFormatRegistry registry, - ShardPath shardPath, Map dataformatAwareStoreHandles) { + ShardPath shardPath, Map dataformatAwareStoreHandles, IndexSettings indexSettings) { } diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java index 3e2a642eda398..230510bb73dca 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java @@ -273,7 +273,7 @@ public void testGetReaderManagers() throws IOException { DataFormatRegistry registry = new DataFormatRegistry(pluginsService); Map> managers = registry.getReaderManager( - new ReaderManagerConfig(Optional.empty(), format, registry, shardPath, Map.of()) + new ReaderManagerConfig(Optional.empty(), format, registry, shardPath, Map.of(), null) ); assertEquals(1, managers.size()); assertNotNull(managers.get(format)); From f44f2c8291af6a02646c0ab35a59c3f70a9ea987 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Sat, 6 Jun 2026 22:25:51 +0530 Subject: [PATCH 09/11] Add NotEqualsSerializer and skip-Lucene rule for `col != ''` perf peer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires `NOT_EQUALS` end-to-end as a Lucene-delegable predicate, then ensures the planner doesn't pay for a useless Lucene round-trip on the canonical empty-string shape. Lucene side: `NotEqualsSerializer` emits the only DSL form OpenSearch has for `!=` — `bool { mustNot: term {...} }`. Registered in `QuerySerializerRegistry` and added to `LuceneAnalyticsBackendPlugin.STANDARD_OPS`. Restricted to keyword fields via a new `NOT_EQUALS_TYPES = {KEYWORD}` because the mustNot-term form silently match-alls on analyzed (text) fields with the standard analyzer. Planner side: a new post-PlanForker pass `PerfPeerNarrowing` walks each `StagePlan`'s resolved fragment, finds `AnnotatedPredicate` leaves with non-empty `performanceDelegationBackends`, and applies registered rules. Each rule is a single-purpose class implementing `PerfPeerNarrowingRule` with a stable name. The first rule `DropLucenePeerForNotEqualsEmptyStringRule` drops Lucene as a perf peer for `col != ''` when the driver is DataFusion — the mustNot-term-empty bitset matches every keyword doc, so AND-intersecting it into the candidate set prunes nothing while costing one FFM round-trip per RG. The pass does not touch `viableBackends`, so Lucene-as-driver alternatives (e.g. count fast-path for `count(*) WHERE col != ''`) are unaffected and continue to win where they should via `BackendShardPreference`. Adds `AnnotatedPredicate#rebuildWithPeers` since `narrowTo` derives peers from `viableBackends` and can't express "narrowed AND zero peers". Tests: `NotEqualsSerializer` round-trip via `QuerySerializerRegistryTests`, `PerfPeerNarrowingTests` for the rule firing on `!= ''` and not on `!= 'foo'`. Signed-off-by: Arpit Bandejiya --- .../lucene/LuceneAnalyticsBackendPlugin.java | 21 +- .../be/lucene/QuerySerializerRegistry.java | 4 +- .../serializers/NotEqualsSerializer.java | 83 ++++++++ .../lucene/QuerySerializerRegistryTests.java | 147 ++++++++++++- .../analytics/exec/DefaultPlanExecutor.java | 6 + ...LucenePeerForNotEqualsEmptyStringRule.java | 94 +++++++++ .../planner/dag/PerfPeerNarrowing.java | 197 ++++++++++++++++++ .../planner/dag/PerfPeerNarrowingRule.java | 46 ++++ .../planner/rel/AnnotatedPredicate.java | 18 ++ .../planner/dag/PerfPeerNarrowingTests.java | 177 ++++++++++++++++ 10 files changed, 787 insertions(+), 6 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/serializers/NotEqualsSerializer.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DropLucenePeerForNotEqualsEmptyStringRule.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowing.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingRule.java create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingTests.java diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java index 9acc87d222e42..feb3f82d1bed7 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java @@ -63,13 +63,13 @@ public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugi // registered in QuerySerializerRegistry — declaring a capability without a matching // DelegatedPredicateSerializer makes the marking layer pick Lucene as viable for // operators it can't actually translate, and the failure surfaces at convert time as - // an IllegalStateException ("No Lucene serializer for [..]"). Today only EQUALS has - // a serializer; range ops, NOT_EQUALS, IS_NULL, IS_NOT_NULL, IN, LIKE are deferred + // an IllegalStateException ("No Lucene serializer for [..]"). Today EQUALS and + // NOT_EQUALS have serializers; range ops, IS_NULL, IS_NOT_NULL, IN, LIKE are deferred // until their serializers land. // TODO: have CapabilityRegistry intersect declared FilterCapability against the // backend's serializer keyset at startup so this list can't drift again. The TODO in // OpenSearchFilterRule.resolveViableBackends references the same constraint. - private static final Set STANDARD_OPS = Set.of(ScalarFunction.EQUALS); + private static final Set STANDARD_OPS = Set.of(ScalarFunction.EQUALS, ScalarFunction.NOT_EQUALS); private static final Set FULL_TEXT_OPS = Set.of( ScalarFunction.MATCH, @@ -98,6 +98,15 @@ public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugi STANDARD_TYPES.add(FieldType.MATCH_ONLY_TEXT); } + // NOT_EQUALS-eligible field types — strictly narrower than {@link #STANDARD_TYPES}. + // The serializer emits {@code bool { mustNot: term { field: value } }}. On a text / + // match_only_text field with the standard analyzer, {@code term: ""} matches nothing + // (analyzer produces zero tokens from the empty string), which makes + // {@code mustNot match-nothing} degenerate to "match all" — wrong results for the + // canonical {@code SearchPhrase != ''} ClickBench shape. Keyword fields are not + // analyzed, so the term lookup is exact and the inversion is correct. + private static final Set NOT_EQUALS_TYPES = Set.of(FieldType.KEYWORD); + private static final Set FULL_TEXT_TYPES = new HashSet<>(); static { FULL_TEXT_TYPES.addAll(FieldType.keyword()); @@ -108,7 +117,11 @@ public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugi static { Set caps = new HashSet<>(); for (ScalarFunction op : STANDARD_OPS) { - caps.add(new FilterCapability.Standard(op, STANDARD_TYPES, LUCENE_FORMATS)); + // Per-op type restrictions: NOT_EQUALS is keyword-only because the + // mustNot-term DSL form misbehaves on analyzed (text) fields. See + // {@link #NOT_EQUALS_TYPES} above. + Set typesForOp = op == ScalarFunction.NOT_EQUALS ? NOT_EQUALS_TYPES : STANDARD_TYPES; + caps.add(new FilterCapability.Standard(op, typesForOp, LUCENE_FORMATS)); } for (ScalarFunction op : FULL_TEXT_OPS) { for (FieldType type : FULL_TEXT_TYPES) { diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index 8fe6e698fd6e5..a65a71b9a13e4 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -17,6 +17,7 @@ import org.opensearch.be.lucene.serializers.MatchPhraseSerializer; import org.opensearch.be.lucene.serializers.MatchSerializer; import org.opensearch.be.lucene.serializers.MultiMatchSerializer; +import org.opensearch.be.lucene.serializers.NotEqualsSerializer; import org.opensearch.be.lucene.serializers.QuerySerializer; import org.opensearch.be.lucene.serializers.QueryStringSerializer; import org.opensearch.be.lucene.serializers.SimpleQueryStringSerializer; @@ -42,7 +43,8 @@ final class QuerySerializerRegistry { Map.entry(ScalarFunction.WILDCARD_QUERY, new WildcardQuerySerializer()), Map.entry(ScalarFunction.QUERY, new QuerySerializer()), Map.entry(ScalarFunction.MATCHALL, new MatchAllSerializer()), - Map.entry(ScalarFunction.EQUALS, new EqualsSerializer()) + Map.entry(ScalarFunction.EQUALS, new EqualsSerializer()), + Map.entry(ScalarFunction.NOT_EQUALS, new NotEqualsSerializer()) ); private QuerySerializerRegistry() {} diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/serializers/NotEqualsSerializer.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/serializers/NotEqualsSerializer.java new file mode 100644 index 0000000000000..0ba0f6020aa0e --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/serializers/NotEqualsSerializer.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene.serializers; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.be.lucene.CalciteToOSMapperConversionUtils; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; + +import java.util.List; + +/** + * Serializer for the NOT_EQUALS operator on a single column with a literal value + * (e.g. {@code tag <> 'hello'} or {@code status <> 200}). The dual to + * {@link EqualsSerializer} — used when a {@code !=} predicate is delegated to + * Lucene as a peer (correctness or performance delegation). + * + *

    Expected RexCall shape: {@code <>($colIdx, literal)}. Either operand + * order is accepted ({@code literal <> $colIdx} also works). + * + *

    OpenSearch's DSL has no native "not equals" leaf — the canonical form is + * {@code bool { must_not: term {...} }}. We emit that shape directly, which the + * field's mapper resolves at {@code toQuery(qsc)} time. Type coercion of the + * literal happens via {@link CalciteToOSMapperConversionUtils} so the mapper + * sees plain Java types (Number, String, etc.) rather than Calcite's BigDecimal + * / NlsString wrappers. + * + *

    NULL semantics caveat. SQL's {@code col <> literal} returns + * UNKNOWN (filtered out) when {@code col} is NULL. Lucene's + * {@code must_not term} excludes only matching docs — non-matching including + * missing-field docs are kept. The planner-level NOT_EQUALS handling is + * expected to add an {@code IS NOT NULL} guard upstream when SQL three-valued + * semantics are required; this serializer only translates the equality + * inversion. + */ +public class NotEqualsSerializer extends AbstractQuerySerializer { + + @Override + public QueryBuilder buildQueryBuilder(RexCall call, List fieldStorage) { + if (call.getOperands().size() != 2) { + throw new IllegalArgumentException("NOT_EQUALS expects 2 operands, got " + call.getOperands().size()); + } + RexNode left = call.getOperands().get(0); + RexNode right = call.getOperands().get(1); + + // Either (column, literal) or (literal, column). + RexInputRef columnRef; + RexLiteral valueLit; + if (left instanceof RexInputRef l && right instanceof RexLiteral r) { + columnRef = l; + valueLit = r; + } else if (left instanceof RexLiteral l && right instanceof RexInputRef r) { + columnRef = r; + valueLit = l; + } else { + throw new IllegalArgumentException( + "NOT_EQUALS performance-delegation requires (RexInputRef, RexLiteral); got " + left + " <> " + right + ); + } + + String fieldName = FieldStorageInfo.resolve(fieldStorage, columnRef.getIndex()).getFieldName(); + // Calcite stores literals in canonical types (BigDecimal for ints, NlsString for + // strings, etc.) that the OpenSearch Mapper can't parse directly. Convert at the + // Calcite ↔ OpenSearch boundary so the mapper sees plain Java types. + Object value = CalciteToOSMapperConversionUtils.literalToOpenSearchValue(valueLit); + TermQueryBuilder term = new TermQueryBuilder(fieldName, value); + BoolQueryBuilder bool = QueryBuilders.boolQuery(); + bool.mustNot(term); + return bool; + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java index 0cc7191c7fb71..0abd985b53f02 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java @@ -29,6 +29,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; @@ -40,6 +41,7 @@ import org.opensearch.index.query.QueryStringQueryBuilder; import org.opensearch.index.query.SimpleQueryStringBuilder; import org.opensearch.index.query.SimpleQueryStringFlag; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.query.WildcardQueryBuilder; import org.opensearch.test.OpenSearchTestCase; @@ -64,7 +66,9 @@ public class QuerySerializerRegistryTests extends OpenSearchTestCase { new NamedWriteableRegistry.Entry(QueryBuilder.class, QueryStringQueryBuilder.NAME, QueryStringQueryBuilder::new), new NamedWriteableRegistry.Entry(QueryBuilder.class, SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new), new NamedWriteableRegistry.Entry(QueryBuilder.class, WildcardQueryBuilder.NAME, WildcardQueryBuilder::new), - new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new) + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, BoolQueryBuilder.NAME, BoolQueryBuilder::new) ) ); @@ -966,6 +970,128 @@ public void testSerializeSimpleQueryStringWithSingleFlag() throws IOException { } } + // --- NotEqualsSerializer tests --- + + /** + * NOT_EQUALS must be registered alongside EQUALS so the marking layer can + * advertise Lucene as a viable peer for {@code <>} predicates. + */ + public void testNotEqualsSerializerIsRegistered() { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.NOT_EQUALS); + assertNotNull("NOT_EQUALS serializer must be registered", serializer); + } + + /** + * {@code col <> 'literal'} round-trips into a {@code bool { mustNot: term { col: literal } }}. + * The bool wrapper is the canonical OpenSearch DSL shape for negated equality. + */ + public void testNotEqualsSerializerRoundTripColumnLeft() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.NOT_EQUALS); + assertNotNull("NOT_EQUALS serializer must be registered", serializer); + + RexCall call = buildNotEqualsCall(/* columnFirst */ true, "tag", "hello"); + List fieldStorage = List.of( + new FieldStorageInfo("tag", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue("Deserialized must be BoolQueryBuilder", deserialized instanceof BoolQueryBuilder); + BoolQueryBuilder bool = (BoolQueryBuilder) deserialized; + assertTrue("must clauses must be empty", bool.must().isEmpty()); + assertEquals("exactly one mustNot clause expected", 1, bool.mustNot().size()); + QueryBuilder mustNotChild = bool.mustNot().get(0); + assertTrue("mustNot child must be a TermQueryBuilder", mustNotChild instanceof TermQueryBuilder); + TermQueryBuilder term = (TermQueryBuilder) mustNotChild; + assertEquals("tag", term.fieldName()); + assertEquals("hello", term.value()); + } + } + + /** + * Operand order shouldn't matter — {@code 'literal' <> col} is the same + * predicate as {@code col <> 'literal'}, so the serializer must accept both. + */ + public void testNotEqualsSerializerAcceptsLiteralLeft() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.NOT_EQUALS); + assertNotNull("NOT_EQUALS serializer must be registered", serializer); + + RexCall call = buildNotEqualsCall(/* columnFirst */ false, "tag", "hello"); + List fieldStorage = List.of( + new FieldStorageInfo("tag", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue("Deserialized must be BoolQueryBuilder", deserialized instanceof BoolQueryBuilder); + BoolQueryBuilder bool = (BoolQueryBuilder) deserialized; + assertEquals("exactly one mustNot clause expected", 1, bool.mustNot().size()); + TermQueryBuilder term = (TermQueryBuilder) bool.mustNot().get(0); + assertEquals("tag", term.fieldName()); + assertEquals("hello", term.value()); + } + } + + /** + * The {@code field <> ''} shape — ClickBench q11/q13/q25/q27/q28 all use this against + * keyword/text fields. {@code STANDARD_TYPES} on the Lucene plugin restricts NOT_EQUALS + * delegation to KEYWORD / TEXT / MATCH_ONLY_TEXT, so this is the realistic round-trip + * shape the marking layer will produce in production. + */ + public void testNotEqualsSerializerRoundTripTextField() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.NOT_EQUALS); + assertNotNull("NOT_EQUALS serializer must be registered", serializer); + + RexCall call = buildNotEqualsCall(/* columnFirst */ true, "SearchPhrase", ""); + List fieldStorage = List.of( + new FieldStorageInfo("SearchPhrase", "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue("Deserialized must be BoolQueryBuilder", deserialized instanceof BoolQueryBuilder); + BoolQueryBuilder bool = (BoolQueryBuilder) deserialized; + assertEquals("exactly one mustNot clause expected", 1, bool.mustNot().size()); + TermQueryBuilder term = (TermQueryBuilder) bool.mustNot().get(0); + assertEquals("SearchPhrase", term.fieldName()); + assertEquals("", term.value()); + } + } + + /** + * Wrong operand shapes (column-column or literal-literal) must throw, not + * silently emit a malformed query. + */ + public void testNotEqualsSerializerRejectsTwoColumns() { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.NOT_EQUALS); + assertNotNull("NOT_EQUALS serializer must be registered", serializer); + + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + RexNode left = rexBuilder.makeInputRef(varcharType, 0); + RexNode right = rexBuilder.makeInputRef(varcharType, 1); + RexCall call = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, left, right); + + List fieldStorage = List.of( + new FieldStorageInfo("a", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false), + new FieldStorageInfo("b", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> serializer.serialize(call, fieldStorage) + ); + assertTrue( + "Exception message must mention NOT_EQUALS, got: " + exception.getMessage(), + exception.getMessage().contains("NOT_EQUALS") + ); + } + // --- Helper methods --- /** @@ -1075,4 +1201,23 @@ private RexCall buildMultiFieldRexCallWithParams( return (RexCall) rexBuilder.makeCall(sqlFunction, operands); } + + /** + * Builds a {@code NOT_EQUALS($colIdx, literal)} {@link RexCall} for serializer + * tests. {@code columnFirst} controls operand order: {@code true} produces + * {@code col <> literal}, {@code false} produces {@code literal <> col}. + * Both shapes are accepted by {@code NotEqualsSerializer} per its javadoc. + */ + private RexCall buildNotEqualsCall(boolean columnFirst, String fieldName, String literalValue) { + // Field name is unused in the RexCall itself — only the column index matters. + // The caller passes a matching FieldStorageInfo at index 0 so resolve(...) finds the name. + assert fieldName != null; + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + RexNode columnRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode literal = rexBuilder.makeLiteral(literalValue); + if (columnFirst) { + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, columnRef, literal); + } + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, literal, columnRef); + } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index 77761d314d739..739d64803e0fe 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -37,6 +37,7 @@ import org.opensearch.analytics.planner.dag.DAGBuilder; import org.opensearch.analytics.planner.dag.FragmentConversionDriver; import org.opensearch.analytics.planner.dag.PlanAlternativeSelector; +import org.opensearch.analytics.planner.dag.PerfPeerNarrowing; import org.opensearch.analytics.planner.dag.PlanForker; import org.opensearch.analytics.planner.dag.QueryDAG; import org.opensearch.analytics.settings.AnalyticsApproximationSettings; @@ -241,6 +242,11 @@ private void executeInternal( final String fullPlan = profile ? org.apache.calcite.plan.RelOptUtil.toString(plan) : null; QueryDAG dag = DAGBuilder.build(plan, capabilityRegistry, clusterService, indexNameExpressionResolver); PlanForker.forkAll(dag, capabilityRegistry); + // Narrow performance-delegation peers per StagePlan based on registered rules. The pass + // doesn't touch viableBackends — backend-driver alternatives produced by PlanForker + // (e.g. Lucene-as-driver count fast-path) are unaffected. Only the DataFusion-driver + // alternative's perf-peer attachments are filtered. See PerfPeerNarrowing javadoc. + PerfPeerNarrowing.narrowAll(dag); BackendPlanAdapter.adaptAll(dag, capabilityRegistry); // Collapse multi-backend stages to a single chosen alternative before conversion // so the convertor runs once per stage and the wire request carries one PlanAlternative. diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DropLucenePeerForNotEqualsEmptyStringRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DropLucenePeerForNotEqualsEmptyStringRule.java new file mode 100644 index 0000000000000..adefc92a31e34 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/DropLucenePeerForNotEqualsEmptyStringRule.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.util.NlsString; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; + +/** + * {@link PerfPeerNarrowingRule} that drops Lucene as a performance-delegation peer + * for predicates of shape {@code col != ''} when the driver is DataFusion. + * + *

    Why. A {@code !=}-against-empty-string predicate matches almost every + * document in a typical index (only docs whose value is exactly the empty string + * fail the test). Asking Lucene for a bitset of matching docs and AND-intersecting + * it into the candidate set buys nothing: the bitset is approximately the entire + * doc-id range, so no candidates get pruned. The cost of the FFM round-trip per + * row group is paid for no benefit. Worse, on text fields the analyzer produces + * zero tokens from the empty string, so {@code mustNot term {field: ""}} + * degenerates to "match all" — strictly wrong filtering — but the keyword-only + * {@link org.opensearch.analytics.spi.FilterCapability} restriction blocks that + * shape from reaching the runtime. + * + *

    Lucene-as-driver alternatives are unaffected: this rule only fires when the + * stage's resolved driver is DataFusion and Lucene appears as a perf peer. + * Stage alternatives where Lucene is the chosen driver (e.g. {@code count(*) + * WHERE col != ''} fast path via {@code IndexSearcher.count()}) are scored + * independently by {@code LuceneShardPreference} and still win where they + * should — this pass doesn't see them as having a "peer" relationship. + * + * @opensearch.internal + */ +final class DropLucenePeerForNotEqualsEmptyStringRule implements PerfPeerNarrowingRule { + + private static final String NAME = "drop-lucene-perf-peer-for-not-equals-empty-string"; + + private final String driverBackend; + private final String peerBackend; + + /** Production wiring: driver = "datafusion", peer = "lucene". */ + DropLucenePeerForNotEqualsEmptyStringRule() { + this("datafusion", "lucene"); + } + + /** Test wiring: caller passes mock backend names ("mock-parquet", "mock-lucene"). */ + DropLucenePeerForNotEqualsEmptyStringRule(String driverBackend, String peerBackend) { + this.driverBackend = driverBackend; + this.peerBackend = peerBackend; + } + + @Override + public String name() { + return NAME; + } + + @Override + public boolean shouldDropPeer(String driverBackend, String peerBackend, AnnotatedPredicate predicate, RexCall original) { + if (!this.driverBackend.equals(driverBackend) || !this.peerBackend.equals(peerBackend)) { + return false; + } + if (original.getKind() != SqlKind.NOT_EQUALS) { + return false; + } + // Either operand may be the literal — `col != ''` and `'' != col` are both valid Calcite + // shapes. The predicate's other operand is typically a RexInputRef but we don't care here; + // any presence of an empty-string literal under NOT_EQUALS is enough for the rule. + for (RexNode operand : original.getOperands()) { + if (operand instanceof RexLiteral lit && isEmptyString(lit)) { + return true; + } + } + return false; + } + + private static boolean isEmptyString(RexLiteral lit) { + Object value = lit.getValue(); + if (value instanceof NlsString nls) { + return nls.getValue().isEmpty(); + } + if (value instanceof String s) { + return s.isEmpty(); + } + return false; + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowing.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowing.java new file mode 100644 index 0000000000000..f2fbcc70b7b08 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowing.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Per-{@link StagePlan} pass that narrows performance-delegation peers on + * {@link AnnotatedPredicate} leaves where the runtime peer call buys nothing. + * Runs between {@link PlanForker#forkAll(QueryDAG, org.opensearch.analytics.planner.CapabilityRegistry)} + * and {@link BackendPlanAdapter#adaptAll(QueryDAG, org.opensearch.analytics.planner.CapabilityRegistry)}. + * + *

    For each stage alternative, walks the resolved fragment's + * {@link OpenSearchFilter}s; for each {@link AnnotatedPredicate} whose + * {@code performanceDelegationBackends} is non-empty, polls every registered + * {@link PerfPeerNarrowingRule}. If any rule returns {@code true} for a given + * peer, that peer is removed. Predicate's {@code viableBackends} stays intact — + * the driver still evaluates natively, the peer just isn't consulted at runtime. + * + *

    The pass is idempotent: once a peer is dropped its rule wouldn't see it + * again (the iterator-style narrowing rebuilds a new {@link AnnotatedPredicate} + * each pass). Adding more rules later is one new class plus one entry in + * {@link #DEFAULT_RULES}. + * + * @opensearch.internal + */ +public final class PerfPeerNarrowing { + + private static final Logger LOGGER = LogManager.getLogger(PerfPeerNarrowing.class); + + private static final List DEFAULT_RULES = List.of(new DropLucenePeerForNotEqualsEmptyStringRule()); + + private PerfPeerNarrowing() {} + + /** Apply the default rule set to every {@link StagePlan} alternative in the DAG. */ + public static void narrowAll(QueryDAG dag) { + narrowAll(dag, DEFAULT_RULES); + } + + /** Test-only entry point: run a custom rule set against the DAG. */ + static void narrowAll(QueryDAG dag, List rules) { + narrowStage(dag.rootStage(), rules); + } + + private static void narrowStage(Stage stage, List rules) { + for (Stage child : stage.getChildStages()) { + narrowStage(child, rules); + } + if (stage.getPlanAlternatives().isEmpty()) { + return; + } + List updated = new ArrayList<>(stage.getPlanAlternatives().size()); + for (StagePlan plan : stage.getPlanAlternatives()) { + RelNode narrowed = narrowFragment(plan.resolvedFragment(), plan.backendId(), rules); + if (narrowed == plan.resolvedFragment()) { + updated.add(plan); + } else { + updated.add(new StagePlan(narrowed, plan.backendId())); + } + } + stage.setPlanAlternatives(updated); + } + + private static RelNode narrowFragment(RelNode node, String driverBackend, List rules) { + List newInputs = new ArrayList<>(node.getInputs().size()); + boolean inputsChanged = false; + for (RelNode child : node.getInputs()) { + RelNode narrowed = narrowFragment(child, driverBackend, rules); + newInputs.add(narrowed); + if (narrowed != child) inputsChanged = true; + } + + if (node instanceof OpenSearchFilter filter) { + RexNode newCondition = narrowCondition(filter.getCondition(), driverBackend, rules); + if (newCondition != filter.getCondition() || inputsChanged) { + return new OpenSearchFilter( + filter.getCluster(), + filter.getTraitSet(), + inputsChanged ? newInputs.getFirst() : filter.getInput(), + newCondition, + filter.getViableBackends() + ); + } + return filter; + } + + return inputsChanged ? node.copy(node.getTraitSet(), newInputs) : node; + } + + private static RexNode narrowCondition(RexNode node, String driverBackend, List rules) { + if (node instanceof AnnotatedPredicate predicate) { + return narrowPredicate(predicate, driverBackend, rules); + } + if (node instanceof RexCall call) { + List newOperands = new ArrayList<>(call.getOperands().size()); + boolean changed = false; + for (RexNode operand : call.getOperands()) { + RexNode narrowed = narrowCondition(operand, driverBackend, rules); + newOperands.add(narrowed); + if (narrowed != operand) changed = true; + } + return changed ? call.clone(call.getType(), newOperands) : call; + } + return node; + } + + private static RexNode narrowPredicate(AnnotatedPredicate predicate, String driverBackend, List rules) { + List peers = predicate.getPerformanceDelegationBackends(); + if (peers.isEmpty()) { + return predicate; + } + if (!(predicate.unwrap() instanceof RexCall original)) { + return predicate; + } + List remaining = new ArrayList<>(peers.size()); + boolean changed = false; + for (String peer : peers) { + boolean drop = false; + for (PerfPeerNarrowingRule rule : rules) { + if (rule.shouldDropPeer(driverBackend, peer, predicate, original)) { + LOGGER.debug( + "PerfPeerNarrowing rule [{}] dropped peer [{}] for predicate id={} (driver=[{}])", + rule.name(), + peer, + predicate.getAnnotationId(), + driverBackend + ); + drop = true; + break; + } + } + if (drop) { + changed = true; + } else { + remaining.add(peer); + } + } + if (!changed) { + return predicate; + } + return rebuildPredicate(predicate, remaining); + } + + /** + * Rebuilds an {@link AnnotatedPredicate} with a new performance-delegation peer list. + * {@code viableBackends} stays intact — the driver hasn't changed; we're just removing + * peers whose runtime consultation isn't useful for this predicate shape. + * + *

    Goes through {@link AnnotatedPredicate#narrowTo(String)} when the result has zero + * peers and the predicate was already narrowed to one viable backend ({@code PlanForker}'s + * standard output); that path is the existing code's "single-viable" shape. Otherwise + * builds a fresh annotation directly. Either way the resulting node is functionally a + * single-viable predicate with no peer. + */ + private static RexNode rebuildPredicate(AnnotatedPredicate predicate, List remainingPeers) { + // narrowTo always produces non-empty peers when the source has multiple viable backends. + // We need an explicit constructor path for the empty-peers case. + return new AnnotatedPredicateBuilder().build( + predicate.getType(), + predicate.unwrap(), + predicate.getViableBackends(), + predicate.getAnnotationId(), + remainingPeers + ); + } + + /** + * Tiny helper to invoke {@link AnnotatedPredicate}'s package-private 5-arg constructor + * via its public-but-shaped-for-this APIs. Today the constructor is private; this stub + * exists so the rebuild path is in one place if we change construction in the future. + * + *

    For now, since only {@link AnnotatedPredicate#clone} preserves all five fields, + * we synthesize the rebuild via {@link AnnotatedPredicate}'s public surface plus a + * narrow proxy in the same package. + */ + private static final class AnnotatedPredicateBuilder { + RexNode build(RelDataType type, RexNode original, List viableBackends, int annotationId, List peers) { + return AnnotatedPredicate.rebuildWithPeers(type, original, viableBackends, annotationId, peers); + } + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingRule.java new file mode 100644 index 0000000000000..203fedd3946c9 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingRule.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.rex.RexCall; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; + +/** + * Decides whether a registered performance-delegation peer should be dropped from a + * given {@link AnnotatedPredicate} on a specific {@link StagePlan}. Used by + * {@link PerfPeerNarrowing} to remove peers whose runtime consultation buys nothing + * (or actively costs more than it saves) for the predicate's shape. + * + *

    Each rule is scoped to a (driver backend, peer backend) pair and a predicate + * shape; if it returns {@code true}, the peer is dropped from + * {@link AnnotatedPredicate#getPerformanceDelegationBackends()} for that one + * predicate on that one stage alternative. + * + *

    Importantly, this does NOT touch {@link AnnotatedPredicate#getViableBackends()}. + * Backend-driver alternatives produced by {@link PlanForker} (e.g. Lucene-as-driver + * count fast-path) are unaffected — those alternatives have their own resolved + * predicates with the chosen backend in {@code viableBackends}, and this pass only + * narrows perf-peers when the chosen driver is the parent stage backend. + * + * @opensearch.internal + */ +public interface PerfPeerNarrowingRule { + + /** + * Stable identifier for logs / tests, e.g. + * {@code "drop-lucene-perf-peer-for-not-equals-empty-string"}. + */ + String name(); + + /** + * Returns {@code true} to drop {@code peerBackend} from {@code predicate}'s + * performance-delegation backends on a stage whose driver is {@code driverBackend}. + */ + boolean shouldDropPeer(String driverBackend, String peerBackend, AnnotatedPredicate predicate, RexCall original); +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java index da0eb5972bce0..9bcb19e5d6e95 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java @@ -84,6 +84,24 @@ private AnnotatedPredicate( this.performanceDelegationBackends = performanceDelegationBackends; } + /** + * Rebuilds an {@link AnnotatedPredicate} with an explicit performance-delegation peer + * list (including the empty list). Used by post-PlanForker passes that drop a peer + * for a specific predicate shape without touching {@code viableBackends} — + * {@link #narrowTo(String)} alone can't express "narrowed AND zero peers" since it + * derives peers from {@code viableBackends}. Constructor stays private; callers must + * route through this factory so the construction surface remains explicit. + */ + public static AnnotatedPredicate rebuildWithPeers( + RelDataType type, + RexNode original, + List viableBackends, + int annotationId, + List performanceDelegationBackends + ) { + return new AnnotatedPredicate(type, original, viableBackends, annotationId, performanceDelegationBackends); + } + public RexNode getOriginal() { return original; } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingTests.java new file mode 100644 index 0000000000000..91e8b32163e82 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/PerfPeerNarrowingTests.java @@ -0,0 +1,177 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.planner.BasePlannerRulesTests; +import org.opensearch.analytics.planner.MockDataFusionBackend; +import org.opensearch.analytics.planner.MockLuceneBackend; +import org.opensearch.analytics.planner.PlannerContext; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Tests for {@link PerfPeerNarrowing} — verifies that the + * {@link DropLucenePeerForNotEqualsEmptyStringRule} drops Lucene as a + * performance-delegation peer for {@code col != ''} on the DataFusion-driver + * stage alternative, while leaving non-empty {@code !=} predicates untouched + * and not affecting the Lucene-driver alternative. + */ +public class PerfPeerNarrowingTests extends BasePlannerRulesTests { + + /** Keyword field with doc values duplicated in both parquet and lucene formats. */ + private Map duplicatedKeywordFields() { + return Map.of( + "tag", + new FieldStorageInfo( + "tag", + "keyword", + FieldType.KEYWORD, + List.of(MockDataFusionBackend.PARQUET_DATA_FORMAT, MockLuceneBackend.LUCENE_DATA_FORMAT), + List.of(), + List.of(), + false + ) + ); + } + + private RexNode makeNotEqualsLiteral(int fieldIndex, String literal) { + return rexBuilder.makeCall( + SqlStdOperatorTable.NOT_EQUALS, + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), fieldIndex), + rexBuilder.makeLiteral(literal) + ); + } + + private QueryDAG buildAndFork(RelNode logicalPlan) { + PlannerContext context = buildContextWithExplicitStorage(1, duplicatedKeywordFields(), List.of(DATAFUSION, LUCENE)); + RelNode optimized = runPlanner(logicalPlan, context); + QueryDAG dag = DAGBuilder.build(optimized, context.getCapabilityRegistry(), mockClusterService(), TEST_RESOLVER); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + return dag; + } + + private org.apache.calcite.plan.RelOptTable keywordTable() { + return mockTable("test_index", new String[] { "tag" }, new SqlTypeName[] { SqlTypeName.VARCHAR }); + } + + /** Returns the AnnotatedPredicate inside the StagePlan's resolved filter, or null. */ + private static AnnotatedPredicate findPredicate(StagePlan plan) { + RelNode current = plan.resolvedFragment(); + while (current != null && current instanceof OpenSearchFilter == false) { + if (current.getInputs().isEmpty()) return null; + current = current.getInputs().getFirst(); + } + if (!(current instanceof OpenSearchFilter filter)) return null; + List annotations = new ArrayList<>(); + collect(filter.getCondition(), annotations); + return annotations.isEmpty() ? null : annotations.getFirst(); + } + + private static void collect(RexNode node, List out) { + if (node instanceof AnnotatedPredicate ap) { + out.add(ap); + return; + } + if (node instanceof org.apache.calcite.rex.RexCall call) { + for (RexNode operand : call.getOperands()) { + collect(operand, out); + } + } + } + + public void testDropsLucenePeerOnDfDriverForNotEqualsEmptyString() { + QueryDAG dag = buildAndFork(LogicalFilter.create(stubScan(keywordTable()), makeNotEqualsLiteral(0, ""))); + + // Pre-narrow assertion — DF alternative exists with Lucene as a perf peer. + StagePlan dfPlanBefore = pickPlan(dag.rootStage(), MockDataFusionBackend.NAME); + AnnotatedPredicate dfPredicateBefore = findPredicate(dfPlanBefore); + assertNotNull("expected an AnnotatedPredicate on DF alternative", dfPredicateBefore); + assertEquals( + "DF predicate should have Lucene as a perf peer pre-narrow", + List.of(MockLuceneBackend.NAME), + dfPredicateBefore.getPerformanceDelegationBackends() + ); + + // Run the narrowing pass with a test-local rule wired to the mock backend names. + PerfPeerNarrowing.narrowAll( + dag, + List.of(new DropLucenePeerForNotEqualsEmptyStringRule(MockDataFusionBackend.NAME, MockLuceneBackend.NAME)) + ); + + StagePlan dfPlanAfter = pickPlan(dag.rootStage(), MockDataFusionBackend.NAME); + AnnotatedPredicate dfPredicateAfter = findPredicate(dfPlanAfter); + assertNotNull(dfPredicateAfter); + assertTrue( + "DF predicate's perf peer should be dropped for != '' " + dfPredicateAfter.getPerformanceDelegationBackends(), + dfPredicateAfter.getPerformanceDelegationBackends().isEmpty() + ); + assertEquals( + "viableBackends must remain narrowed to DF", + List.of(MockDataFusionBackend.NAME), + dfPredicateAfter.getViableBackends() + ); + + // If a Lucene-driver alternative exists (e.g. for count fast-path shapes), the rule + // must not have rewritten its predicate — it only fires when driver is DF + peer is Lucene. + StagePlan lucenePlanAfter = findPlan(dag.rootStage(), MockLuceneBackend.NAME); + if (lucenePlanAfter != null) { + AnnotatedPredicate lucenePredicateAfter = findPredicate(lucenePlanAfter); + assertNotNull(lucenePredicateAfter); + assertEquals( + "Lucene-driver alternative must not be narrowed", + List.of(MockLuceneBackend.NAME), + lucenePredicateAfter.getViableBackends() + ); + } + } + + public void testKeepsLucenePeerForNonEmptyLiteral() { + QueryDAG dag = buildAndFork(LogicalFilter.create(stubScan(keywordTable()), makeNotEqualsLiteral(0, "hello"))); + StagePlan dfPlanBefore = pickPlan(dag.rootStage(), MockDataFusionBackend.NAME); + AnnotatedPredicate before = findPredicate(dfPlanBefore); + assertEquals(List.of(MockLuceneBackend.NAME), before.getPerformanceDelegationBackends()); + + PerfPeerNarrowing.narrowAll( + dag, + List.of(new DropLucenePeerForNotEqualsEmptyStringRule(MockDataFusionBackend.NAME, MockLuceneBackend.NAME)) + ); + + StagePlan dfPlanAfter = pickPlan(dag.rootStage(), MockDataFusionBackend.NAME); + AnnotatedPredicate after = findPredicate(dfPlanAfter); + assertEquals( + "non-empty literal must keep Lucene as a perf peer", + List.of(MockLuceneBackend.NAME), + after.getPerformanceDelegationBackends() + ); + } + + private static StagePlan pickPlan(Stage stage, String backendId) { + StagePlan plan = findPlan(stage, backendId); + if (plan == null) throw new AssertionError("no plan alternative for backend " + backendId); + return plan; + } + + private static StagePlan findPlan(Stage stage, String backendId) { + for (StagePlan plan : stage.getPlanAlternatives()) { + if (plan.backendId().equals(backendId)) return plan; + } + return null; + } +} From c96dd420353ee7f5e64c7acd96e97b416f6cf48c Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Sat, 6 Jun 2026 23:44:09 +0530 Subject: [PATCH 10/11] Lucene Weight.count gate for performance-delegation peer consultation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second gate before the Rust driver consults Lucene as a peer for a performance-delegated leaf. After the existing 5% page-pruning gate, the driver asks Lucene for a cheap per-segment selectivity estimate; if the bitset would approximate match-all, the FFM round-trip is skipped because AND-intersection wouldn't prune anything. Java side: * New SPI hook FilterDelegationHandle#estimateSelectivityPpm returning selectivity in parts-per-million on `[0, 1_000_000]`, or `-1` for unknown. * LuceneFilterDelegationHandle decomposes each provider's compiled Weight into per-leaf LeafCountEstimators at createProvider time. Two concrete shapes: - WeightCountEstimator delegates to Lucene's Weight.count fast-paths (TermQuery → docFreq, MatchAllDocsQuery → numDocs, PointRangeQuery → BKD). - MustNotTermEstimator does numDocs - docFreq via a single term-dictionary lookup, since Lucene's BooleanWeight.count returns -1 for any query containing a MUST_NOT clause and that's the canonical NotEqualsSerializer output. At runtime, estimateSelectivityPpm walks each leaf's estimators, takes the conjunction-bounded minimum, and normalises by the segment's numDocs. * Wired into the FFM upcall surface (df_register_filter_tree_callbacks now takes 6 callbacks; FilterTreeCallbacks adds estimateSelectivityPpm). * New cluster setting analytics.lucene.peer_consultation_threshold (NodeScope, Dynamic, default 0.10) plumbed through WireConfigSnapshot to the Rust driver. 0.0 = never consult on this gate, 1.0 = always consult. * WireConfigSnapshot grows a double field (offset 72, BYTE_SIZE 80). Rust side: * DatafusionQueryConfig + WireDatafusionQueryConfig pick up the threshold; clamped to [0,1] in from_wire as ABI hygiene. * SingleCollectorEvaluator stores the threshold; prefetch_rg now runs both gates in order. When the second gate skips, candidates fall through to the same materialisation as the consult path so DF runs the residual natively. * New estimate_selectivity_ppm helper wraps the FFM upcall, returning Option. Tests: LeafCountEstimatorsTests covers TermQuery, mustNot-term (present and absent term), all-MUST conjunction, and the SHOULD-fallback-to-Weight.count case. WireConfigSnapshot/DatafusionSettings test counts updated for the new field/setting. Signed-off-by: Arpit Bandejiya --- .../analytics/spi/FilterDelegationHandle.java | 25 +++ .../rust/src/datafusion_query_config.rs | 19 ++ .../rust/src/indexed_executor.rs | 2 + .../indexed_table/eval/single_collector.rs | 73 ++++++-- .../rust/src/indexed_table/ffm_callbacks.rs | 36 +++- .../tests_e2e/fuzz/delegation.rs | 1 + .../indexed_table/tests_e2e/fuzz/harness.rs | 1 + .../indexed_table/tests_e2e/multi_segment.rs | 2 + .../indexed_table/tests_e2e/page_pruning.rs | 1 + .../rust/src/query_executor.rs | 2 +- .../be/datafusion/DatafusionSettings.java | 30 +++- .../be/datafusion/WireConfigSnapshot.java | 26 ++- .../indexfilter/FilterTreeCallbacks.java | 33 ++++ .../be/datafusion/nativelib/NativeBridge.java | 17 +- .../DataFusionPluginSettingsTests.java | 2 +- .../datafusion/DatafusionSettingsTests.java | 2 +- .../datafusion/WireConfigSnapshotTests.java | 2 +- .../be/lucene/LeafCountEstimator.java | 37 ++++ .../be/lucene/LeafCountEstimators.java | 164 ++++++++++++++++++ .../lucene/LuceneFilterDelegationHandle.java | 73 +++++++- .../be/lucene/LeafCountEstimatorsTests.java | 137 +++++++++++++++ 21 files changed, 661 insertions(+), 24 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimator.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimators.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LeafCountEstimatorsTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java index 9fcffabd4fb6d..bae6a83676a17 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java @@ -88,4 +88,29 @@ public interface FilterDelegationHandle extends Closeable { default boolean isCancelled() { return false; } + + /** + * Cheap, approximate selectivity for the provider on one segment, returned in + * parts-per-million ({@code [0, 1_000_000]}) so a single i64 carries it across + * the FFM boundary without floating-point. Drives the driver-side + * peer-consultation gate: when the selectivity exceeds the configured + * threshold, the driver skips the peer call (the bitset would approximate + * match-all and AND-intersection wouldn't prune anything). + * + *

    The implementation derives the count via per-leaf minimum over the + * conjunction the provider represents — a conjunction's count is bounded by + * the smallest leaf's count. If any leaf can't be estimated, returns + * {@code -1} ("unknown") so the driver consults to be safe. + * + *

    Default returns {@code -1} so backends without an estimate path don't + * change driver behaviour. + * + * @param providerKey the key returned from {@link #createProvider(int)} + * @param writerGeneration the writer generation identifying the segment + * @return selectivity in parts-per-million in {@code [0, 1_000_000]}, + * or {@code -1} if unknown + */ + default long estimateSelectivityPpm(int providerKey, long writerGeneration) { + return -1L; + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs index 05bab3321128b..70e6412a9a1ab 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs @@ -72,6 +72,14 @@ pub struct DatafusionQueryConfig { pub query_strategy: QueryStrategy, /// Whether to use bloom filters for row group pruning on the indexed read path. pub bloom_filter_on_read: bool, + /// Threshold for the driver-side Lucene peer-consultation gate. When DF page + /// pruning kept >5% of an RG (existing first gate), the driver asks Lucene + /// for a cheap document-count estimate; if the estimate over the segment's + /// total docs is at most this fraction, the peer is consulted (its bitset + /// is selective enough to narrow further). Otherwise the peer call is + /// skipped. `0.0` = never consult; `1.0` = always consult on this gate. + /// Set via `analytics.lucene.peer_consultation_threshold` (default `0.10`). + pub lucene_peer_consultation_threshold: f64, } /// FFM wire format. Must stay in lockstep with the Java `MemoryLayout`. @@ -105,6 +113,9 @@ pub struct WireDatafusionQueryConfig { pub query_strategy: i32, /// 0 = false, 1 = true pub bloom_filter_on_read: i32, + /// Threshold (0..1) for the Lucene peer-consultation second gate. See + /// `DatafusionQueryConfig::lucene_peer_consultation_threshold`. + pub lucene_peer_consultation_threshold: f64, } impl DatafusionQueryConfig { @@ -129,6 +140,7 @@ impl DatafusionQueryConfig { tree_collector_strategy: CollectorCallStrategy::TightenOuterBounds, query_strategy: QueryStrategy::None, bloom_filter_on_read: true, + lucene_peer_consultation_threshold: 0.10, } } @@ -200,6 +212,9 @@ impl DatafusionQueryConfig { _ => QueryStrategy::None, }, bloom_filter_on_read: w.bloom_filter_on_read != 0, + // Clamp to [0.0, 1.0]. Java validates the cluster setting in the same range, so + // out-of-range values shouldn't cross the wire — clamp defensively for ABI hygiene. + lucene_peer_consultation_threshold: w.lucene_peer_consultation_threshold.clamp(0.0, 1.0), } } } @@ -316,6 +331,7 @@ mod tests { tree_collector_strategy: 1, query_strategy: 1, bloom_filter_on_read: 1, + lucene_peer_consultation_threshold: 0.25, }; let ptr = &wire as *const _ as i64; let c = unsafe { DatafusionQueryConfig::from_ffm_ptr(ptr) }; @@ -330,6 +346,7 @@ mod tests { assert_eq!(c.cost_predicate, 3); assert_eq!(c.cost_collector, 17); assert_eq!(c.query_strategy, QueryStrategy::ListingTable); + assert!((c.lucene_peer_consultation_threshold - 0.25).abs() < 1e-9); } #[test] @@ -350,11 +367,13 @@ mod tests { tree_collector_strategy: 1, query_strategy: 0, bloom_filter_on_read: 0, + lucene_peer_consultation_threshold: 0.10, }; let ptr = &wire as *const _ as i64; let c = unsafe { DatafusionQueryConfig::from_ffm_ptr(ptr) }; assert_eq!(c.force_strategy, None); assert_eq!(c.force_pushdown, None); assert_eq!(c.query_strategy, QueryStrategy::None); + assert!((c.lucene_peer_consultation_threshold - 0.10).abs() < 1e-9); } } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs index d368305f625bf..c0e4c1cabf667 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs @@ -692,6 +692,7 @@ async unsafe fn execute_indexed_with_context_inner( let bloom_store = Arc::clone(&store); let bloom_schema = schema.clone(); let bloom_on_read = query_config.bloom_filter_on_read; + let lucene_peer_consultation_threshold = query_config.lucene_peer_consultation_threshold; Arc::new( move |segment: &SegmentFileInfo, chunk, stream_metrics: &StreamMetrics| { let collector_opt: Option> = match &correctness_provider { @@ -749,6 +750,7 @@ async unsafe fn execute_indexed_with_context_inner( Arc::new(crate::indexed_table::eval::single_collector::FfmDelegatedBackendCollectorFactory), context_id, bloom_config, + lucene_peer_consultation_threshold, )); Ok(eval) }, diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs index ab897f4360ff4..bd138c3b7133f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs @@ -31,7 +31,7 @@ use native_bridge_common::log_debug; use roaring::RoaringBitmap; use super::{PrefetchedRg, RowGroupBitsetSource}; -use crate::indexed_table::ffm_callbacks::{create_provider, FfmSegmentCollector, ProviderHandle}; +use crate::indexed_table::ffm_callbacks::{create_provider, estimate_selectivity_ppm, FfmSegmentCollector, ProviderHandle}; use crate::indexed_table::index::RowGroupDocsCollector; use crate::indexed_table::page_pruner::{PagePruneMetrics, PagePruner}; use crate::indexed_table::row_selection::{ @@ -45,12 +45,10 @@ use std::time::Instant; pub use super::CollectorCallStrategy; use crate::indexed_table::stream::RowGroupInfo; -/// TODO(phase-99): hardcoded selectivity threshold for opportunistic peer consultation. -/// Replaced by a cluster setting plumbed through `WireConfigSnapshot` and -/// `DatafusionQueryConfig` in the very last phase, after Phase 7 OR/NOT support and -/// everything else. Until then, performance-delegated leaves consult the peer when DF -/// page-pruning kept more than 5% of an RG. -const HARDCODED_SELECTIVITY_THRESHOLD: f64 = 0.05; +/// First gate threshold: when DF page pruning keeps MORE than this fraction of an RG, +/// page pruning wasn't selective enough and we proceed to gate 2 (Lucene estimate). +/// When DF already squeezed below this, the peer call is skipped. +const PAGE_PRUNE_SELECTIVITY_THRESHOLD: f64 = 0.05; /// Builds delegated-backend collectors for performance-delegated leaves. Production impl /// wraps `FfmSegmentCollector::create` (Java/Lucene round-trip); fuzz tests inject a @@ -177,6 +175,11 @@ pub struct SingleCollectorEvaluator { context_id: i64, /// Bloom filter pruning config. None = disabled. bloom_config: Option, + /// Lucene peer-consultation second gate. After page pruning kept >5% of an RG, ask + /// Lucene for a cheap doc-count estimate; consult only when the estimate over the + /// segment's total docs is at most this fraction. `0.0` = never consult on this + /// gate. `1.0` = always consult. Plumbed through `DatafusionQueryConfig`. + lucene_peer_consultation_threshold: f64, } /// Resources needed for per-RG bloom filter pruning. @@ -191,6 +194,7 @@ pub struct BloomConfig { } impl SingleCollectorEvaluator { + #[allow(clippy::too_many_arguments)] pub fn new( collector: Option>, page_pruner: Arc, @@ -204,6 +208,7 @@ impl SingleCollectorEvaluator { delegated_backend_collector_factory: Arc, context_id: i64, bloom_config: Option, + lucene_peer_consultation_threshold: f64, ) -> Self { Self { collector, @@ -218,6 +223,7 @@ impl SingleCollectorEvaluator { delegated_backend_collector_factory, context_id, bloom_config, + lucene_peer_consultation_threshold, } } } @@ -394,15 +400,22 @@ impl RowGroupBitsetSource for SingleCollectorEvaluator { } }; - // Opportunistic peer consultation for performance-delegated leaves. Only fires - // when DF page-pruning kept more than the configured fraction of the RG — - // skipping the FFM round-trip when DF was already selective. Lazy: lock the - // map only if the gate fires; create the provider only once per query × leaf. + // Opportunistic peer consultation for performance-delegated leaves. Two gates: + // 1. DF page-pruning selectivity — if DF already pruned ≤ 5% of the RG, the + // peer call is skipped (DF is selective enough on its own). + // 2. Lucene cheap doc-count estimate — when (1) didn't help, ask Lucene for an + // estimate via Weight.count + per-leaf min over the conjunction. Consult + // only when the estimate over the segment's total docs is at most the + // configured threshold; otherwise the bitset would approximate match-all + // and AND-intersection wouldn't prune anything. Threshold sourced from + // `analytics.lucene.peer_consultation_threshold` (default 0.10). + // Lazy throughout — provider is created only once per query × leaf, and only + // when the gates fire. // TODO(d3): consult ALL performance leaves whose gate fires and AND their // bitsets. Today we consult the first leaf only — sufficient for AND-only // single-call demo. Multi-leaf intersection is part of D3 follow-up. if !self.performance_provider_locks.is_empty() - && should_consult_lucene(&page_ranges, rg, HARDCODED_SELECTIVITY_THRESHOLD) + && should_consult_lucene(&page_ranges, rg, PAGE_PRUNE_SELECTIVITY_THRESHOLD) { // Pick the smallest annotation_id deterministically so logs/tests are stable. // Avoids the Vec/sort allocation in the common single-leaf case. @@ -435,6 +448,37 @@ impl RowGroupBitsetSource for SingleCollectorEvaluator { ); } + // Second gate: ask Lucene for a cheap per-segment selectivity estimate via + // Weight.count + per-leaf min over the conjunction. Skip the FFM round-trip + // when the peer's bitset would approximate match-all (estimated selectivity + // exceeds the configured threshold). `None` means "Java couldn't cheaply + // derive it" → consult anyway (safe default). + let threshold_ppm = (self.lucene_peer_consultation_threshold * 1_000_000.0) as i64; + let consult = match estimate_selectivity_ppm(context_id, provider.key(), self.writer_generation) { + Some(ppm) => ppm <= threshold_ppm, + None => true, + }; + if !consult { + // Drop this AND-leaf's contribution: candidates already reflect DF page-pruning. + // Skip ahead to the post-peer materialization without intersecting a peer bitset. + let mask_len = rg.num_rows as usize; + let packed_bits = bitmap_to_packed_bits(&candidates, mask_len as u32); + let mask_buffer = datafusion::arrow::buffer::Buffer::from_vec(packed_bits); + if candidates.is_empty() { + return Ok(None); + } + return Ok(Some(PrefetchedRg { + candidates: candidates.clone(), + eval_nanos: t.elapsed().as_nanos() as u64, + context: Box::new(SingleCollectorState { + candidates, + mask_buffer: mask_buffer.clone(), + mask_len, + }), + mask_buffer: Some(mask_buffer), + })); + } + let collector = self .delegated_backend_collector_factory .create(context_id, provider.key(), self.writer_generation, min_doc, max_doc) @@ -691,6 +735,7 @@ mod tests { }) as Arc; let pruner = minimal_page_pruner(); let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, None); + let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, 0.10); let rg = RowGroupInfo { index: 0, @@ -707,6 +752,7 @@ mod tests { let collector = Arc::new(StubCollector { docs: vec![0] }) as Arc; let pruner = minimal_page_pruner(); let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, None); + let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, 0.10); let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); let batch = datafusion::arrow::record_batch::RecordBatch::try_new( schema, @@ -735,6 +781,7 @@ mod tests { let collector = Arc::new(StubCollector { docs: vec![0] }) as Arc; let pruner = minimal_page_pruner(); let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, None); + let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, 0.10); assert!(eval.needs_row_mask()); } @@ -743,6 +790,7 @@ mod tests { let collector = Arc::new(StubCollector { docs: vec![] }) as Arc; let pruner = minimal_page_pruner(); let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, None); + let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, 0.10); let rg = RowGroupInfo { index: 0, first_row: 0, @@ -763,6 +811,7 @@ mod tests { }) as Arc; let pruner = minimal_page_pruner(); let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, None); + let eval = SingleCollectorEvaluator::new(Some(collector), pruner, None, None, None, None, CollectorCallStrategy::FullRange, Arc::new(HashMap::new()), 0, Arc::new(FfmDelegatedBackendCollectorFactory), 0, 0.10); let rg = RowGroupInfo { index: 0, diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs index 7810b35054745..93ea811d74843 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs @@ -8,7 +8,7 @@ //! FFM upcall surface for index-filter providers and collectors. //! -//! Five callback slots, populated once at startup by +//! Six callback slots, populated once at startup by //! `df_register_filter_tree_callbacks` (see `ffm.rs`): //! //! - `createProvider(contextId, annotationId) -> providerKey|-1` @@ -16,6 +16,9 @@ //! - `collectDocs(contextId, collectorKey, minDoc, maxDoc, outBuf, outWordCap) -> wordsWritten|-1` //! - `releaseCollector(contextId, collectorKey)` //! - `releaseProvider(contextId, providerKey)` +//! - `estimateSelectivityPpm(contextId, providerKey, writerGeneration) -> ppm|-1` — +//! cheap per-segment selectivity (count / segment-doc-count) in parts-per-million +//! ([0, 1_000_000]) consumed by the driver-side peer-consultation gate. //! //! `ProviderHandle` and `FfmSegmentCollector` are the lifetime wrappers — //! they call the release callbacks on drop. @@ -39,12 +42,18 @@ type ReleaseProviderFn = unsafe extern "C" fn(i64, i32); type CreateCollectorFn = unsafe extern "C" fn(i64, i32, i64, i32, i32) -> i32; type CollectDocsFn = unsafe extern "C" fn(i64, i32, i32, i32, *mut u64, i64) -> i64; type ReleaseCollectorFn = unsafe extern "C" fn(i64, i32); +/// `(context_id, provider_key, writer_generation) -> ppm | -1`. Cheap per-segment +/// selectivity (count / segment-doc-count) in parts-per-million in `[0, 1_000_000]`, +/// used by the driver-side peer-consultation gate. `-1` means "I can't tell +/// quickly" — driver will fall back to consulting. +type EstimateSelectivityPpmFn = unsafe extern "C" fn(i64, i32, i64) -> i64; static CREATE_PROVIDER: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); static RELEASE_PROVIDER: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); static CREATE_COLLECTOR: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); static COLLECT_DOCS: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); static RELEASE_COLLECTOR: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); +static ESTIMATE_SELECTIVITY_PPM: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); /// Registered by Java at startup. Stores function pointers into atomic /// slots. Each call to this entry replaces the slots wholesale. @@ -59,6 +68,7 @@ pub unsafe extern "C" fn df_register_filter_tree_callbacks( create_collector: CreateCollectorFn, collect_docs: CollectDocsFn, release_collector: ReleaseCollectorFn, + estimate_selectivity_ppm: EstimateSelectivityPpmFn, ) { // catch_unwind is defense-in-depth: atomic stores shouldn't panic, // but if they ever did (e.g. allocator OOM if we grew the atomics), @@ -71,6 +81,7 @@ pub unsafe extern "C" fn df_register_filter_tree_callbacks( CREATE_COLLECTOR.store(create_collector as *mut (), Ordering::Release); COLLECT_DOCS.store(collect_docs as *mut (), Ordering::Release); RELEASE_COLLECTOR.store(release_collector as *mut (), Ordering::Release); + ESTIMATE_SELECTIVITY_PPM.store(estimate_selectivity_ppm as *mut (), Ordering::Release); })); } @@ -111,6 +122,29 @@ fn load_release_collector() -> Option { Some(unsafe { std::mem::transmute::<*mut (), ReleaseCollectorFn>(p) }) } } +fn load_estimate_selectivity_ppm() -> Option { + let p = ESTIMATE_SELECTIVITY_PPM.load(Ordering::Acquire); + if p.is_null() { + None + } else { + Some(unsafe { std::mem::transmute::<*mut (), EstimateSelectivityPpmFn>(p) }) + } +} + +/// Cheap per-segment selectivity (count / segment-doc-count) in parts-per-million +/// in `[0, 1_000_000]`, used by the driver-side peer-consultation gate. Returns +/// `Some(ppm)` on a non-negative result; `None` when the Java side returns `-1` +/// (couldn't cheaply derive) or the callback slot wasn't registered. The caller +/// treats `None` as "I don't know — be conservative and consult". +pub fn estimate_selectivity_ppm(context_id: i64, provider_key: i32, writer_generation: i64) -> Option { + let f = load_estimate_selectivity_ppm()?; + let n = unsafe { f(context_id, provider_key, writer_generation) }; + if n < 0 { + None + } else { + Some(n) + } +} // ── ProviderHandle — owns `releaseProvider` on drop ─────────────────── diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/delegation.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/delegation.rs index 2534e79921150..892747ef0271f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/delegation.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/delegation.rs @@ -423,6 +423,7 @@ pub(in crate::indexed_table::tests_e2e) async fn execute_delegation_tree( Arc::clone(&factory), 0, None, + 0.10, )); Ok(eval) }) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs index 3999f051fa337..1a43addfd1587 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs @@ -403,6 +403,7 @@ pub(in crate::indexed_table::tests_e2e) async fn execute_tree_single_collector( std::sync::Arc::new(crate::indexed_table::eval::single_collector::FfmDelegatedBackendCollectorFactory), 0, None, + 0.10, )); let _ = segment; Ok(eval) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs index 40fbee4f0564e..7a99915970397 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs @@ -161,6 +161,7 @@ async fn run_two_segment_query( std::sync::Arc::new(crate::indexed_table::eval::single_collector::FfmDelegatedBackendCollectorFactory), 0, None, + 0.10, ), ); Ok(eval) @@ -571,6 +572,7 @@ async fn run_segments(specs: Vec, num_partitions: usize) -> Vec<(i32, S std::sync::Arc::new(crate::indexed_table::eval::single_collector::FfmDelegatedBackendCollectorFactory), 0, None, + 0.10, ), ); Ok(eval) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs index d01a50ea95a4b..c148d5ac8bf79 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs @@ -363,6 +363,7 @@ async fn run_single_collector( std::sync::Arc::new(crate::indexed_table::eval::single_collector::FfmDelegatedBackendCollectorFactory), 0, None, + 0.10, )); Ok(eval) }) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index 5e242dd2d44ad..9e174550ed682 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -305,7 +305,7 @@ pub async fn execute_with_context( // and fall through to the standard decode + execute below. if let Some(prepared) = handle.prepared_plan.as_ref() { let physical_plan = std::sync::Arc::clone(prepared); - let df_stream = execute_stream(physical_plan, handle.ctx.task_ctx()).map_err(|e| { + let df_stream = execute_stream(Arc::clone(&physical_plan), handle.ctx.task_ctx()).map_err(|e| { error!("execute_with_context: failed to execute prepared plan: {}", e); e })?; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java index 499fe3bf141d2..25731ff6e00f9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java @@ -236,6 +236,27 @@ public final class DatafusionSettings { Setting.Property.Dynamic ); + /** + * Threshold [0.0, 1.0] for the Lucene peer-consultation second gate. The driver-side + * {@code SingleCollectorEvaluator} runs DataFusion's page-pruning first; if page + * pruning kept more than the existing 5% of an RG, it asks Lucene for a cheap + * per-segment doc-count estimate (via {@code Weight.count} + a per-leaf min over the + * conjunction). The peer is consulted only when the estimate over the segment's + * total docs is at most this fraction; otherwise the peer call is skipped because + * its bitset wouldn't narrow further. + *

    + * {@code 0.0} = never consult on this gate. {@code 1.0} = always consult. + * Default: {@code 0.10}. + */ + public static final Setting LUCENE_PEER_CONSULTATION_THRESHOLD = Setting.doubleSetting( + "analytics.lucene.peer_consultation_threshold", + 0.10, + 0.0, + 1.0, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + // ── All settings registered by the plugin ── public static final List> ALL_SETTINGS = List.of( @@ -273,7 +294,8 @@ public final class DatafusionSettings { INDEXED_SINGLE_COLLECTOR_STRATEGY, INDEXED_TREE_COLLECTOR_STRATEGY, INDEXED_MAX_COLLECTOR_PARALLELISM, - INDEXED_QUERY_STRATEGY + INDEXED_QUERY_STRATEGY, + LUCENE_PEER_CONSULTATION_THRESHOLD ); // ── Snapshot management ── @@ -316,6 +338,7 @@ public DatafusionSettings(ClusterService clusterService) { .treeCollectorStrategy(strategyToWireValue(INDEXED_TREE_COLLECTOR_STRATEGY.get(settings))) .maxCollectorParallelism(INDEXED_MAX_COLLECTOR_PARALLELISM.get(settings)) .queryStrategy(queryStrategyToWireValue(INDEXED_QUERY_STRATEGY.get(settings))) + .lucenePeerConsultationThreshold(LUCENE_PEER_CONSULTATION_THRESHOLD.get(settings)) .build(); registerListeners(clusterSettings); @@ -340,6 +363,7 @@ public DatafusionSettings(ClusterService clusterService) { .treeCollectorStrategy(strategyToWireValue(INDEXED_TREE_COLLECTOR_STRATEGY.get(settings))) .maxCollectorParallelism(INDEXED_MAX_COLLECTOR_PARALLELISM.get(settings)) .queryStrategy(queryStrategyToWireValue(INDEXED_QUERY_STRATEGY.get(settings))) + .lucenePeerConsultationThreshold(LUCENE_PEER_CONSULTATION_THRESHOLD.get(settings)) .build(); } @@ -380,6 +404,10 @@ void registerListeners(ClusterSettings clusterSettings) { snapshot = WireConfigSnapshot.builder(snapshot).queryStrategy(queryStrategyToWireValue(newValue)).build(); }); + clusterSettings.addSettingsUpdateConsumer(LUCENE_PEER_CONSULTATION_THRESHOLD, newValue -> { + snapshot = WireConfigSnapshot.builder(snapshot).lucenePeerConsultationThreshold(newValue).build(); + }); + clusterSettings.addSettingsUpdateConsumer(SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING, newValue -> { this.maxSliceCount = newValue; snapshot = WireConfigSnapshot.builder(snapshot) diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java index 915f8ddd10c30..3159c250ecec1 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java @@ -26,7 +26,7 @@ public final class WireConfigSnapshot { /** Total byte size of the wire struct ({@code WireDatafusionQueryConfig}). */ - public static final long BYTE_SIZE = 76; + public static final long BYTE_SIZE = 88; private final int batchSize; private final int targetPartitions; @@ -38,6 +38,7 @@ public final class WireConfigSnapshot { private final int singleCollectorStrategy; private final int treeCollectorStrategy; private final int queryStrategy; + private final double lucenePeerConsultationThreshold; private WireConfigSnapshot(Builder builder) { this.batchSize = builder.batchSize; @@ -50,6 +51,7 @@ private WireConfigSnapshot(Builder builder) { this.singleCollectorStrategy = builder.singleCollectorStrategy; this.treeCollectorStrategy = builder.treeCollectorStrategy; this.queryStrategy = builder.queryStrategy; + this.lucenePeerConsultationThreshold = builder.lucenePeerConsultationThreshold; } public static Builder builder() { @@ -70,7 +72,8 @@ public static Builder builder(WireConfigSnapshot current) { .maxCollectorParallelism(current.maxCollectorParallelism) .singleCollectorStrategy(current.singleCollectorStrategy) .treeCollectorStrategy(current.treeCollectorStrategy) - .queryStrategy(current.queryStrategy); + .queryStrategy(current.queryStrategy) + .lucenePeerConsultationThreshold(current.lucenePeerConsultationThreshold); } public int batchSize() { @@ -113,6 +116,10 @@ public int queryStrategy() { return queryStrategy; } + public double lucenePeerConsultationThreshold() { + return lucenePeerConsultationThreshold; + } + /** * Writes this snapshot into a {@code MemorySegment} matching the * {@code WireDatafusionQueryConfig} {@code #[repr(C)]} layout. @@ -138,8 +145,10 @@ public int queryStrategy() { * 64 4 tree_collector_strategy i32 from snapshot * 68 4 query_strategy i32 from snapshot (0/1/2) * 72 4 bloom_filter_on_read i32 from snapshot (0/1) + * 76 4 (padding for 8-byte alignment of next f64) + * 80 8 lucene_peer_consultation_threshold f64 from snapshot ([0.0, 1.0]) * ────── ──── - * Total: 76 bytes + * Total: 88 bytes * * * @param segment the target memory segment (at least {@link #BYTE_SIZE} bytes) @@ -175,6 +184,11 @@ public void writeTo(MemorySegment segment) { segment.set(ValueLayout.JAVA_INT, 68, queryStrategy); // Offset 72: bloom_filter_on_read (i32) — 0 = false, 1 = true segment.set(ValueLayout.JAVA_INT, 72, bloomFilterOnRead ? 1 : 0); + // Offset 76-79: 4 bytes of padding so the next f64 starts on an 8-byte boundary. + // Rust's `#[repr(C)]` would insert this padding automatically; we mirror it explicitly + // here so MemorySegment aligned-writes don't fault. + // Offset 80: lucene_peer_consultation_threshold (f64) — [0.0, 1.0] + segment.set(ValueLayout.JAVA_DOUBLE, 80, lucenePeerConsultationThreshold); } /** @@ -192,6 +206,7 @@ public static final class Builder { private int singleCollectorStrategy = 2; // PageRangeSplit private int treeCollectorStrategy = 1; // TightenOuterBounds private int queryStrategy = 2; // IndexedPredicateOnly (matches DatafusionSettings default "indexed") + private double lucenePeerConsultationThreshold = 0.10; private Builder() {} @@ -245,6 +260,11 @@ public Builder queryStrategy(int queryStrategy) { return this; } + public Builder lucenePeerConsultationThreshold(double lucenePeerConsultationThreshold) { + this.lucenePeerConsultationThreshold = lucenePeerConsultationThreshold; + return this; + } + public WireConfigSnapshot build() { return new WireConfigSnapshot(this); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java index a3c86cc019333..1d18b4703c655 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java @@ -260,6 +260,39 @@ public static long collectDocs(long contextId, int collectorKey, int minDoc, int } } + /** + * {@code estimateSelectivityPpm(contextId, providerKey, writerGeneration) -> ppm|-1}. + * Returns selectivity (count / segment-doc-count) in parts-per-million in + * {@code [0, 1_000_000]}, or {@code -1} when unknown. Used by the driver's + * peer-consultation gate. + */ + public static long estimateSelectivityPpm(long contextId, int providerKey, long writerGeneration) { + long tid = trackStart(contextId); + try { + QueryBinding binding = BINDINGS.get(contextId); + assertBindingExists(binding, "estimateSelectivityPpm", contextId); + if (binding == null || binding.handle() == null) { + return -1L; + } + return binding.handle().estimateSelectivityPpm(providerKey, writerGeneration); + } catch (AssertionError e) { + throw e; + } catch (Throwable throwable) { + LOGGER.error( + new ParameterizedMessage( + "estimateSelectivityPpm(contextId={}, providerKey={}, writerGeneration={}) failed", + contextId, + providerKey, + writerGeneration + ), + throwable + ); + return -1L; + } finally { + trackEnd(contextId, tid); + } + } + /** * {@code releaseCollector(contextId, collectorKey)}. Never throws. */ diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index e3b224f9406f9..401ba0dfd1202 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -347,7 +347,8 @@ public final class NativeBridge { ) ); - // void df_register_filter_tree_callbacks(createCollector, collectDocs, releaseCollector) + // void df_register_filter_tree_callbacks(createProvider, releaseProvider, createCollector, + // collectDocs, releaseCollector, estimateSelectivityPpm) REGISTER_FILTER_TREE_CALLBACKS = linker.downcallHandle( lib.find("df_register_filter_tree_callbacks").orElseThrow(), FunctionDescriptor.ofVoid( @@ -355,6 +356,7 @@ public final class NativeBridge { ValueLayout.ADDRESS, ValueLayout.ADDRESS, ValueLayout.ADDRESS, + ValueLayout.ADDRESS, ValueLayout.ADDRESS ) ); @@ -595,6 +597,11 @@ private static void installFilterTreeCallbacks(Linker linker) { "releaseCollector", java.lang.invoke.MethodType.methodType(void.class, long.class, int.class) ); + MethodHandle estimateSelectivityPpm = lookup.findStatic( + cb, + "estimateSelectivityPpm", + java.lang.invoke.MethodType.methodType(long.class, long.class, int.class, long.class) + ); java.lang.foreign.MemorySegment createProviderStub = linker.upcallStub( createProvider, @@ -636,13 +643,19 @@ private static void installFilterTreeCallbacks(Linker linker) { FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT), arena ); + java.lang.foreign.MemorySegment estimateSelectivityPpmStub = linker.upcallStub( + estimateSelectivityPpm, + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG), + arena + ); NativeCall.invokeVoid( REGISTER_FILTER_TREE_CALLBACKS, createProviderStub, releaseProviderStub, createCollectorStub, collectDocsStub, - releaseCollectorStub + releaseCollectorStub, + estimateSelectivityPpmStub ); } catch (Throwable t) { throw new ExceptionInInitializerError(t); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java index ffac7f875165f..4ba9c91e611be 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java @@ -114,7 +114,7 @@ public void testGetSettingsReturnsAllIndexedSettings() { public void testGetSettingsReturnsTotalExpectedCount() { try (DataFusionPlugin plugin = new DataFusionPlugin()) { List> settings = plugin.getSettings(); - assertEquals(27, settings.size()); + assertEquals(28, settings.size()); } catch (Exception e) { throw new AssertionError(e); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java index 097b1b63c1616..644acd680509a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java @@ -69,7 +69,7 @@ public void testMaxCollectorParallelismSettingDefinition() { } public void testAllSettingsContainsAllExpectedSettings() { - assertEquals(27, DatafusionSettings.ALL_SETTINGS.size()); + assertEquals(28, DatafusionSettings.ALL_SETTINGS.size()); assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DataFusionPlugin.DATAFUSION_REDUCE_TARGET_PARTITIONS)); assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DataFusionPlugin.DATAFUSION_SPILL_DIRECTORY)); assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_BATCH_SIZE)); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java index 02de2481daa9c..24112ecedce83 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java @@ -17,7 +17,7 @@ public class WireConfigSnapshotTests extends OpenSearchTestCase { public void testByteSize() { - assertEquals(76L, WireConfigSnapshot.BYTE_SIZE); + assertEquals(88L, WireConfigSnapshot.BYTE_SIZE); } public void testWriteToWritesCorrectValuesAtCorrectOffsets() { diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimator.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimator.java new file mode 100644 index 0000000000000..2da7315ca0299 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimator.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.lucene.index.LeafReaderContext; + +import java.io.IOException; + +/** + * Returns a cheap, approximate document count for one Lucene query shape on one + * segment leaf. Built once at handle construction (per delegated leaf), evaluated + * per call to {@link LuceneFilterDelegationHandle#estimateSelectivityPpm(int, long)}. + * + *

    Implementations dispatch on the underlying {@code Query} subtype: + *

      + *
    • plain {@link org.apache.lucene.search.TermQuery} → {@link org.apache.lucene.search.Weight#count(LeafReaderContext)} + * (returns {@code docFreq} from the term dictionary)
      • + *
    • {@code BooleanQuery} of shape {@code mustNot term} → {@code numDocs - docFreq} + * via a single term-dictionary lookup (Lucene's own {@code BooleanWeight.count} + * returns {@code -1} for any query containing a {@code MUST_NOT} clause)
      • + *
    • anything else → returns {@code -1} so the driver consults to be safe
      • + *
    + */ +interface LeafCountEstimator { + /** + * Returns a non-negative count of matching documents in {@code leaf}, or + * {@code -1} if the count can't be cheaply derived. Implementations must be + * idempotent and free of per-leaf caching state — they're called per RG. + */ + long estimate(LeafReaderContext leaf) throws IOException; +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimators.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimators.java new file mode 100644 index 0000000000000..ef48d6c585037 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LeafCountEstimators.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; + +import java.io.IOException; +import java.util.List; + +/** + * Static factory + concrete {@link LeafCountEstimator} implementations. + * + *

    {@link #buildList(Weight, Query)} decomposes a compiled query into a list of + * per-leaf estimators — one per AND-clause when the top-level shape is a + * {@link BooleanQuery} of MUST clauses, or a single estimator for an atomic shape. + * The driver evaluates every estimator on a leaf and takes the minimum, since a + * conjunction's count is bounded by the smallest leaf's count. + */ +final class LeafCountEstimators { + + private LeafCountEstimators() {} + + /** + * Build the per-leaf estimators for one delegated provider's compiled query. + * Returns a singleton list for non-conjunctive shapes; one entry per MUST + * clause for top-level {@code BooleanQuery} of all-MUST clauses. + * + *

    Each returned estimator is independent — the runtime evaluates them on a + * leaf and takes {@code min}; any {@code -1} poisons the result to {@code -1}. + */ + static List buildList(Weight weight, Query rewrittenQuery) { + if (rewrittenQuery instanceof BooleanQuery bq && isAllMust(bq)) { + // Each MUST clause becomes its own estimator. The compiled Weight covers the + // full bool query, so for the Weight.count fast-path we still pass the parent + // weight for clauses we don't recognize individually — Lucene's BooleanWeight + // happens to short-circuit -1 anyway, so they degenerate to "unknown" which is + // the safe answer. Recognized leaf shapes (TermQuery, mustNot-term BoolQuery) + // get their own dedicated estimators that bypass BooleanWeight.count. + java.util.ArrayList result = new java.util.ArrayList<>(bq.clauses().size()); + for (BooleanClause clause : bq.clauses()) { + result.add(forSingleQuery(clause.query(), weight)); + } + return result; + } + return List.of(forSingleQuery(rewrittenQuery, weight)); + } + + private static boolean isAllMust(BooleanQuery bq) { + for (BooleanClause clause : bq.clauses()) { + if (clause.occur() != BooleanClause.Occur.MUST) { + return false; + } + } + return true; + } + + /** + * Pick the right estimator for one (possibly nested) query shape. Falls back to + * the parent {@link Weight}'s {@code count} when the shape doesn't match a + * dedicated path — which itself returns {@code -1} for any + * {@link BooleanQuery} carrying a {@code MUST_NOT} clause, so the driver + * defaults to consulting in that case. + */ + private static LeafCountEstimator forSingleQuery(Query q, Weight parentWeight) { + if (q instanceof TermQuery tq) { + return new WeightCountEstimator(parentWeight); + } + Term mustNotTerm = extractMustNotTerm(q); + if (mustNotTerm != null) { + return new MustNotTermEstimator(mustNotTerm); + } + return new WeightCountEstimator(parentWeight); + } + + /** + * If {@code q} has shape {@code BooleanQuery { mustNot: TermQuery }} (with + * exactly that one clause), returns the inner term. Otherwise {@code null}. + * The {@link org.opensearch.be.lucene.serializers.NotEqualsSerializer} emits + * exactly this shape for {@code col != literal} on keyword fields — keeping + * the matcher narrow keeps a future serializer change from accidentally + * activating this path on a different shape that happens to contain + * {@code MUST_NOT}. + */ + private static Term extractMustNotTerm(Query q) { + if (q instanceof BooleanQuery bq && bq.clauses().size() == 1) { + BooleanClause clause = bq.clauses().get(0); + if (clause.occur() == BooleanClause.Occur.MUST_NOT && clause.query() instanceof TermQuery tq) { + return tq.getTerm(); + } + } + return null; + } + + /** + * Delegates to {@link Weight#count(LeafReaderContext)} which Lucene implements + * natively for atomic shapes (TermQuery → docFreq, PointRangeQuery → BKD count, + * MatchAllDocsQuery → numDocs). Returns {@code -1} when Lucene doesn't have a + * fast-path — most notably for any {@link BooleanQuery} containing a + * {@code MUST_NOT}, which is why the {@link MustNotTermEstimator} below + * exists. + */ + static final class WeightCountEstimator implements LeafCountEstimator { + private final Weight weight; + + WeightCountEstimator(Weight weight) { + this.weight = weight; + } + + @Override + public long estimate(LeafReaderContext leaf) throws IOException { + int count = weight.count(leaf); + // Weight.count returns -1 when Lucene has no fast-path (e.g. BooleanWeight + // with a MUST_NOT clause); preserve that so the driver consults. Otherwise + // widen to long (counts are non-negative when known). + return count < 0 ? -1L : count; + } + } + + /** + * Counts {@code numDocs - docFreq(term)} on the leaf — exact for + * {@code mustNot term} on a non-tombstoned term, conservative when the term is + * absent (returns the full segment doc count, which is correct: every doc + * matches "not equal to a value that doesn't exist"). + */ + static final class MustNotTermEstimator implements LeafCountEstimator { + private final Term term; + + MustNotTermEstimator(Term term) { + this.term = term; + } + + @Override + public long estimate(LeafReaderContext leaf) throws IOException { + int total = leaf.reader().numDocs(); + if (total <= 0) { + return 0; + } + Terms terms = leaf.reader().terms(term.field()); + if (terms == null) { + // Field absent on this segment — every doc satisfies `mustNot term(absent)`. + return total; + } + TermsEnum it = terms.iterator(); + if (!it.seekExact(term.bytes())) { + return total; + } + return Math.max(0L, (long) total - (long) it.docFreq()); + } + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java index 9f3e8d3ae0e98..8771d6e21577b 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java @@ -71,6 +71,14 @@ final class LuceneFilterDelegationHandle implements FilterDelegationHandle { private final Map generationToSegmentName; private final ConcurrentHashMap weightsByProviderKey = new ConcurrentHashMap<>(); + /** + * Per-provider list of {@link LeafCountEstimator}s, populated when + * {@link #createProvider(int)} compiles the Weight. Read by + * {@link #estimateSelectivityPpm(int, long)} to derive a cheap estimate for the + * driver-side peer-consultation gate. Same lifecycle as + * {@link #weightsByProviderKey}. + */ + private final ConcurrentHashMap> estimatorsByProviderKey = new ConcurrentHashMap<>(); private final ConcurrentHashMap scorersByCollectorKey = new ConcurrentHashMap<>(); private final AtomicInteger nextProviderKey = new AtomicInteger(1); private final AtomicInteger nextCollectorKey = new AtomicInteger(1); @@ -123,9 +131,15 @@ public int createProvider(int annotationId) { return -1; } try { - Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + Query rewritten = searcher.rewrite(query); + Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE_NO_SCORES, 1.0f); int providerKey = nextProviderKey.getAndIncrement(); weightsByProviderKey.put(providerKey, weight); + // Build per-leaf estimators once per provider so the runtime estimate path is + // allocation-free. The list is per-conjunction-clause for top-level all-MUST + // BooleanQueries; one entry otherwise. Empty list shouldn't be possible — empty + // BooleanQueries are rewritten to MatchNoDocsQuery. + estimatorsByProviderKey.put(providerKey, LeafCountEstimators.buildList(weight, rewritten)); LOGGER.debug("[scf] createProvider annotationId={} → providerKey={}", annotationId, providerKey); return providerKey; } catch (IOException exception) { @@ -208,6 +222,61 @@ public boolean isCancelled() { return isCancelledSupplier != null && isCancelledSupplier.getAsBoolean(); } + @Override + public long estimateSelectivityPpm(int providerKey, long writerGeneration) { + List estimators = estimatorsByProviderKey.get(providerKey); + if (estimators == null || estimators.isEmpty()) { + return -1L; + } + String segName = generationToSegmentName.get(writerGeneration); + if (segName == null) { + return -1L; + } + LeafReaderContext leaf = null; + for (LeafReaderContext lrc : leaves) { + if (unwrapSegmentReader(lrc.reader()).getSegmentInfo().info.name.equals(segName)) { + leaf = lrc; + break; + } + } + if (leaf == null) { + return -1L; + } + int total = leaf.reader().numDocs(); + if (total <= 0) { + // Empty / fully-deleted segment — every fraction is trivially 0; return that + // so the gate treats this segment as maximally selective and consults. + return 0L; + } + try { + // Conjunction count is bounded by the smallest leaf; any leaf returning -1 + // ("can't tell cheaply") poisons the result so the driver consults to be safe. + long min = Long.MAX_VALUE; + for (LeafCountEstimator estimator : estimators) { + long c = estimator.estimate(leaf); + if (c < 0) { + return -1L; + } + if (c < min) { + min = c; + } + } + if (min == Long.MAX_VALUE) { + return -1L; + } + // selectivity = min / total, encoded in parts-per-million for FFM transport. + // Capped at 1_000_000 in case rounding ever pushes a count above total. + long ppm = (min * 1_000_000L) / total; + return Math.min(ppm, 1_000_000L); + } catch (IOException exception) { + LOGGER.warn( + "estimateSelectivityPpm IOException for providerKey=" + providerKey + " writerGeneration=" + writerGeneration, + exception + ); + return -1L; + } + } + @Override public int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out) { ScorerHandle handle = scorersByCollectorKey.get(collectorKey); @@ -268,11 +337,13 @@ public void releaseCollector(int collectorKey) { @Override public void releaseProvider(int providerKey) { weightsByProviderKey.remove(providerKey); + estimatorsByProviderKey.remove(providerKey); } @Override public void close() { weightsByProviderKey.clear(); + estimatorsByProviderKey.clear(); scorersByCollectorKey.clear(); } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LeafCountEstimatorsTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LeafCountEstimatorsTests.java new file mode 100644 index 0000000000000..4c2be67572a60 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LeafCountEstimatorsTests.java @@ -0,0 +1,137 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Verifies the per-leaf count estimators backing + * {@link LuceneFilterDelegationHandle#estimateSelectivityPpm(int, long)}. + * + *

    Index shape: 100 docs, half {@code tag = "hello"}, half {@code tag = "goodbye"}. + * Lets us check both the {@code TermQuery → docFreq} fast path and the + * {@code mustNot term → numDocs - docFreq} fallback, plus the per-leaf-min over + * an all-MUST conjunction. + */ +public class LeafCountEstimatorsTests extends OpenSearchTestCase { + + private Directory directory; + private IndexWriter writer; + private DirectoryReader reader; + private LeafReaderContext leaf; + private IndexSearcher searcher; + + @Override + public void setUp() throws Exception { + super.setUp(); + directory = new ByteBuffersDirectory(); + writer = new IndexWriter(directory, new IndexWriterConfig()); + for (int i = 0; i < 100; i++) { + Document doc = new Document(); + doc.add(new StringField("tag", i % 2 == 0 ? "hello" : "goodbye", Field.Store.NO)); + writer.addDocument(doc); + } + writer.commit(); + reader = DirectoryReader.open(writer); + leaf = reader.leaves().get(0); + searcher = new IndexSearcher(reader); + } + + @Override + public void tearDown() throws Exception { + reader.close(); + writer.close(); + directory.close(); + super.tearDown(); + } + + public void testTermQueryReturnsDocFreq() throws Exception { + Query q = new TermQuery(new Term("tag", "hello")); + Weight weight = searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + List estimators = LeafCountEstimators.buildList(weight, q); + assertEquals(1, estimators.size()); + assertEquals("docFreq must equal half the index", 50L, estimators.get(0).estimate(leaf)); + } + + public void testMustNotTermReturnsNumDocsMinusDocFreq() throws Exception { + BooleanQuery q = new BooleanQuery.Builder().add(new TermQuery(new Term("tag", "hello")), BooleanClause.Occur.MUST_NOT).build(); + Weight weight = searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + List estimators = LeafCountEstimators.buildList(weight, q); + assertEquals(1, estimators.size()); + // Lucene's BooleanWeight.count returns -1 for any query containing MUST_NOT, + // so we must end up with the dedicated MustNotTermEstimator path here, not the + // generic Weight.count fallback. + assertEquals("docs not matching 'hello' = full index minus 50", 50L, estimators.get(0).estimate(leaf)); + } + + public void testMustNotMissingTermReturnsAllDocs() throws Exception { + BooleanQuery q = new BooleanQuery.Builder().add(new TermQuery(new Term("tag", "missing")), BooleanClause.Occur.MUST_NOT).build(); + Weight weight = searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + List estimators = LeafCountEstimators.buildList(weight, q); + assertEquals("term absent from segment → every doc satisfies !=", 100L, estimators.get(0).estimate(leaf)); + } + + public void testAllMustConjunctionExposesPerLeafEstimators() throws Exception { + BooleanQuery q = new BooleanQuery.Builder().add(new TermQuery(new Term("tag", "hello")), BooleanClause.Occur.MUST) + .add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST) + .build(); + Weight weight = searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + List estimators = LeafCountEstimators.buildList(weight, q); + // One estimator per MUST clause. Generic clauses (no dedicated leaf path) + // delegate to the parent BooleanWeight — that's what gives the conjunction- + // bounded result the caller takes the min over. Both estimators here use the + // generic path (TermQuery and MatchAllDocsQuery don't trigger the dedicated + // mustNot-term branch), so each returns the parent's count. + assertEquals("one estimator per MUST clause", 2, estimators.size()); + long expectedConjunction = searcher.count(q); + for (LeafCountEstimator estimator : estimators) { + long c = estimator.estimate(leaf); + // Either Weight.count had a fast-path (returns the conjunction count) or + // gave up (-1 → caller consults). Both are acceptable; the rule is no + // false positives. + assertTrue("estimate must be -1 or " + expectedConjunction + ", got " + c, c == -1L || c == expectedConjunction); + } + } + + public void testNonRecognizedShapeFallsBackToWeightCount() throws Exception { + // BooleanQuery with a SHOULD clause — not all-MUST, so we fall through to the parent + // Weight.count. BooleanWeight.count returns -1 for non-leaf queries it doesn't have a + // fast path for, so the estimator should propagate -1 (caller treats as "consult"). + BooleanQuery q = new BooleanQuery.Builder().add(new TermQuery(new Term("tag", "hello")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term("tag", "goodbye")), BooleanClause.Occur.SHOULD) + .build(); + Weight weight = searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + List estimators = LeafCountEstimators.buildList(weight, q); + assertEquals("non-conjunctive top-level → single fallback estimator", 1, estimators.size()); + long count = estimators.get(0).estimate(leaf); + // Either the BooleanWeight has a count fast-path (rare) — in which case it equals 100 — + // or it returns -1. Both are acceptable; the rule is "don't lie". + assertTrue("count must be -1 (unknown) or full index, got " + count, count == -1L || count == 100L); + } +} From f993a7394fa9241443508a3584861b641fbcfcb6 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Sun, 7 Jun 2026 14:42:50 +0530 Subject: [PATCH 11/11] Add loggers Signed-off-by: Arpit Bandejiya --- .../rust/src/local_executor.rs | 10 ++++++++++ .../rust/src/query_executor.rs | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs index f51835dd7b7f2..433dcab72383a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs @@ -184,8 +184,13 @@ impl LocalSession { DataFusionError::Execution(format!("Failed to decode Substrait plan: {}", e)) })?; let logical_plan = from_substrait_plan(&self.ctx.state(), &plan).await?; + log_debug!("DataFusion logical plan:\n{}", logical_plan.display_indent()); let dataframe = self.ctx.execute_logical_plan(logical_plan).await?; let physical_plan = dataframe.create_physical_plan().await?; + log_debug!( + "DataFusion physical plan:\n{}", + displayable(physical_plan.as_ref()).indent(true) + ); let target_schema = crate::schema_coerce::coerce_inferred_schema(physical_plan.schema()); let physical_plan = crate::relabel_exec::wrap_if_relabel_needed(physical_plan, target_schema)?; @@ -218,8 +223,13 @@ impl LocalSession { )) })?; let logical_plan = from_substrait_plan(&self.ctx.state(), &plan).await?; + log_debug!("DataFusion logical plan:\n{}", logical_plan.display_indent()); let dataframe = self.ctx.execute_logical_plan(logical_plan).await?; let physical_plan = dataframe.create_physical_plan().await?; + log_debug!( + "DataFusion physical plan:\n{}", + displayable(physical_plan.as_ref()).indent(true) + ); // Strip first so `force_aggregate_mode(Final)` can find the Final/Partial pair // through the raw plan; then derive `target_schema` and wrap with RelabelExec from // the stripped output (otherwise the relabel target would carry the pre-strip Final diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index 9e174550ed682..ad99cefcb034e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -368,6 +368,11 @@ pub async fn execute_with_context( let target_schema = crate::schema_coerce::coerce_inferred_schema(physical_plan.schema()); let physical_plan = crate::relabel_exec::wrap_if_relabel_needed(physical_plan, target_schema)?; + log_debug!( + "DataFusion physical plan:\n{}", + displayable(physical_plan.as_ref()).indent(true) + ); + let df_stream = execute_stream(physical_plan.clone(), handle.ctx.task_ctx()).map_err(|e| { error!("execute_with_context: failed to create stream: {}", e); e