diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java index 819b205face2..b1274c6a9d64 100644 --- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java +++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java @@ -399,7 +399,7 @@ public enum CassandraRelevantProperties MONITORING_EXECUTION_INFO_ENABLED("cassandra.monitoring_execution_info_enabled", "true"), /** The current version of the SAI on-disk index format. */ - SAI_CURRENT_VERSION("cassandra.sai.latest.version", "ec"), + SAI_CURRENT_VERSION("cassandra.sai.latest.version", "ed"), /** The class to use for selecting the current version of the SAI on-disk index format on a per-keyspace basis. */ SAI_VERSION_SELECTOR_CLASS("cassandra.sai.version.selector.class", ""), diff --git a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java index 7fe468593b03..cbb8dc3ba839 100644 --- a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java @@ -947,6 +947,11 @@ private String toString(Iterator columns, boolean cql, boolean r while (columns.hasNext()) { ColumnMetadata column = columns.next(); + + // Skip synthetic columns in CQL representation as they are internal implementation details + if (cql && column.isSynthetic()) + continue; + String columnName = cql ? column.name.toCQLString() : String.valueOf(column.name); SortedSet s = subSelections != null diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java index fb317fb0fb61..cb97b326c090 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java +++ b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java @@ -87,6 +87,7 @@ public class Version implements Comparable public static final Version VECTOR_EARLIEST = BA; public static final Version JVECTOR_EARLIEST = CA; public static final Version BM25_EARLIEST = EC; + public static final Version SYNTHETIC_COLUMN_EARLIEST = ED; public static final Version LATEST = ALL.get(0); // This is volatile rather than final so that tests may use reflection to change it and safely publish across threads, diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/SlowSAIQueryLoggerTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/SlowSAIQueryLoggerTest.java index 646eb246d004..94ee076aee62 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/sai/SlowSAIQueryLoggerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/sai/SlowSAIQueryLoggerTest.java @@ -61,6 +61,9 @@ public void testSlowSAIQueryLogger() throws Throwable // effectively disable the scheduled monitoring task so we control it manually for better test stability CassandraRelevantProperties.MONITORING_REPORT_INTERVAL_MS.setInt((int) TimeUnit.HOURS.toMillis(1)); + // enable term statistics, same as in SAITester + CassandraRelevantProperties.SAI_QUERY_OPTIMIZATION_USE_TERM_STATISTICS.setBoolean(true); + try (Cluster cluster = init(Cluster.build(1) .withConfig(c -> c.set("slow_query_log_timeout_in_ms", SLOW_QUERY_LOG_TIMEOUT_IN_MS)) .withInstanceInitializer(BB::install) diff --git a/test/unit/org/apache/cassandra/db/MultiRangeReadCommandCQLTest.java b/test/unit/org/apache/cassandra/db/MultiRangeReadCommandCQLTest.java index 2a15ac49c800..096c6e4a3e42 100644 --- a/test/unit/org/apache/cassandra/db/MultiRangeReadCommandCQLTest.java +++ b/test/unit/org/apache/cassandra/db/MultiRangeReadCommandCQLTest.java @@ -27,6 +27,8 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.index.sai.SAIUtil; +import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.utils.Pair; import org.assertj.core.api.Assertions; @@ -93,18 +95,25 @@ public void testToCQLString() multiRangeError); // test ANN index-based ORDER BY - createTable("CREATE TABLE %s (k int, c int, n int, v vector, PRIMARY KEY (k, c))"); - createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"); - createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'"); - String truncationError = "no viable alternative at input '..'"; - assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10", - "SELECT * FROM %s WHERE (token(k) <= -1 OR token(k) > -1) ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", - "SELECT * FROM %s WHERE (token(k) <= ? OR token(k) > ?) ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", - truncationError); - assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10", - "SELECT * FROM %s WHERE (token(k) <= -1 OR token(k) > -1) AND n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", - "SELECT * FROM %s WHERE (token(k) <= ? OR token(k) > ?) AND n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", - truncationError); + SAIUtil.withVersionsOnOrAfter(Version.JVECTOR_EARLIEST, version -> { + + createTable("CREATE TABLE %s (k int, c int, n int, v vector, PRIMARY KEY (k, c))"); + createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"); + createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'"); + + // the synthetic score column makes us not to send a wildcard column filter but individual column selections + String columns = version.onOrAfter(Version.SYNTHETIC_COLUMN_EARLIEST) ? "n, v" : "*"; + String truncationError = "no viable alternative at input '..'"; + + assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10", + "SELECT " + columns + " FROM %s WHERE (token(k) <= -1 OR token(k) > -1) ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", + "SELECT " + columns + " FROM %s WHERE (token(k) <= ? OR token(k) > ?) ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", + truncationError); + assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10", + "SELECT " + columns + " FROM %s WHERE (token(k) <= -1 OR token(k) > -1) AND n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", + "SELECT " + columns + " FROM %s WHERE (token(k) <= ? OR token(k) > ?) AND n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", + truncationError); + }); } @Override diff --git a/test/unit/org/apache/cassandra/db/PartitionRangeReadCommandCQLTest.java b/test/unit/org/apache/cassandra/db/PartitionRangeReadCommandCQLTest.java index 819a27b6ec93..cc920575cfd5 100644 --- a/test/unit/org/apache/cassandra/db/PartitionRangeReadCommandCQLTest.java +++ b/test/unit/org/apache/cassandra/db/PartitionRangeReadCommandCQLTest.java @@ -26,6 +26,8 @@ import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.index.StubIndex; +import org.apache.cassandra.index.sai.SAIUtil; +import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.assertj.core.api.Assertions; @@ -130,18 +132,25 @@ public void testToCQLString() "SELECT * FROM %s WHERE n = ? ORDER BY n ASC LIMIT 10 ALLOW FILTERING"); // test ANN index-based ORDER BY - createTable("CREATE TABLE %s (k int, c int, n int, v vector, PRIMARY KEY (k, c))"); - createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"); - createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'"); - String truncationError = "no viable alternative at input '..'"; - assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10", - "SELECT * FROM %s ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", - "SELECT * FROM %s ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", - truncationError); - assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10", - "SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", - "SELECT * FROM %s WHERE n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", - truncationError); + SAIUtil.withVersionsOnOrAfter(Version.JVECTOR_EARLIEST, version -> { + + createTable("CREATE TABLE %s (k int, c int, n int, v vector, PRIMARY KEY (k, c))"); + createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"); + createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'"); + + // the synthetic score column makes us not to send a wildcard column filter but individual column selections + String columns = version.onOrAfter(Version.SYNTHETIC_COLUMN_EARLIEST) ? "n, v" : "*"; + String truncationError = "no viable alternative at input '..'"; + + assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10", + "SELECT " + columns + " FROM %s ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", + "SELECT " + columns + " FROM %s ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", + truncationError); + assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10", + "SELECT " + columns + " FROM %s WHERE n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", + "SELECT " + columns + " FROM %s WHERE n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", + truncationError); + }); // test literals createTable("CREATE TABLE %s (k text, c text, m map, PRIMARY KEY (k, c))"); diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java index fd67413fb7f5..5114b208e629 100644 --- a/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java +++ b/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java @@ -23,6 +23,8 @@ import org.junit.Test; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.index.sai.SAIUtil; +import org.apache.cassandra.index.sai.disk.format.Version; import static java.util.Arrays.asList; import static org.junit.Assert.assertTrue; @@ -208,13 +210,20 @@ public void testToCQLString() "SELECT * FROM %s WHERE k = ? AND c IN (?, ?) ORDER BY n ASC LIMIT 10 ALLOW FILTERING"); // test ANN index-based ORDER BY - createTable("CREATE TABLE %s (k int, c int, n int, v vector, PRIMARY KEY (k, c))"); - createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"); - createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'"); - assertToCQLString("SELECT * FROM %s WHERE k = 0 ORDER BY v ANN OF [1, 2] LIMIT 10", - "SELECT * FROM %s WHERE k = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", - "SELECT * FROM %s WHERE k = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", - "no viable alternative at input '..'"); + SAIUtil.withVersionsOnOrAfter(Version.JVECTOR_EARLIEST, version -> { + + createTable("CREATE TABLE %s (k int, c int, n int, v vector, PRIMARY KEY (k, c))"); + createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'"); + createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'"); + + // the synthetic score column makes us not to send a wildcard column filter but individual column selections + String columns = version.onOrAfter(Version.SYNTHETIC_COLUMN_EARLIEST) ? "n, v" : "*"; + + assertToCQLString("SELECT * FROM %s WHERE k = 0 ORDER BY v ANN OF [1, 2] LIMIT 10", + "SELECT " + columns + " FROM %s WHERE k = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING", + "SELECT " + columns + " FROM %s WHERE k = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING", + "no viable alternative at input '..'"); + }); // test with index and multi-column clustering createTable("CREATE TABLE %s (k int, c1 int, c2 int,v int, PRIMARY KEY (k, c1, c2))"); diff --git a/test/unit/org/apache/cassandra/index/sai/SAIUtil.java b/test/unit/org/apache/cassandra/index/sai/SAIUtil.java index ecf45981829e..b873f314f502 100644 --- a/test/unit/org/apache/cassandra/index/sai/SAIUtil.java +++ b/test/unit/org/apache/cassandra/index/sai/SAIUtil.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.Map; +import java.util.function.Consumer; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.cql3.CQLTester; @@ -136,6 +137,32 @@ public static void setEnableFused(boolean enableFused) } } + /** + * Runs the provided tester for each SAI version on or after the provided version. + * + * @param version the version to start testing from + * @param tester a tester consuming the tested version + */ + public static void withVersionsOnOrAfter(Version version, Consumer tester) + { + Version currentVersion = currentVersion(); + try + { + for (Version testedVersion : Version.ALL) + { + if (!testedVersion.onOrAfter(version)) + continue; + + SAIUtil.setCurrentVersion(testedVersion); + tester.accept(testedVersion); + } + } + finally + { + SAIUtil.setCurrentVersion(currentVersion); + } + } + public static class CustomVersionSelector implements Version.Selector { private final Version defaultVersion;