Skip to content

CNDB-15170: Make ED the default SAI index format version #2428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adelapena wants to merge 1 commit into
base: main
Choose a base branch
from
Open

CNDB-15170: Make ED the default SAI index format version #2428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ public enum CassandraRelevantProperties
MONITORING_EXECUTION_INFO_ENABLED("cassandra.monitoring_execution_info_enabled", "true"),

/** The current version of the SAI on-disk index format. */
SAI_CURRENT_VERSION("cassandra.sai.latest.version", "ec"),
SAI_CURRENT_VERSION("cassandra.sai.latest.version", "ed"),

/** The class to use for selecting the current version of the SAI on-disk index format on a per-keyspace basis. */
SAI_VERSION_SELECTOR_CLASS("cassandra.sai.version.selector.class", ""),
Expand Down
5 changes: 5 additions & 0 deletions src/java/org/apache/cassandra/db/filter/ColumnFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -947,6 +947,11 @@ private String toString(Iterator<ColumnMetadata> columns, boolean cql, boolean r
while (columns.hasNext())
{
ColumnMetadata column = columns.next();

// Skip synthetic columns in CQL representation as they are internal implementation details
if (cql && column.isSynthetic())
continue;

String columnName = cql ? column.name.toCQLString() : String.valueOf(column.name);

SortedSet<ColumnSubselection> s = subSelections != null
Expand Down
1 change: 1 addition & 0 deletions src/java/org/apache/cassandra/index/sai/disk/format/Version.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ public class Version implements Comparable<Version>
public static final Version VECTOR_EARLIEST = BA;
public static final Version JVECTOR_EARLIEST = CA;
public static final Version BM25_EARLIEST = EC;
public static final Version SYNTHETIC_COLUMN_EARLIEST = ED;
public static final Version LATEST = ALL.get(0);

// This is volatile rather than final so that tests may use reflection to change it and safely publish across threads,
Expand Down
3 changes: 3 additions & 0 deletions test/distributed/org/apache/cassandra/distributed/test/sai/SlowSAIQueryLoggerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ public void testSlowSAIQueryLogger() throws Throwable
// effectively disable the scheduled monitoring task so we control it manually for better test stability
CassandraRelevantProperties.MONITORING_REPORT_INTERVAL_MS.setInt((int) TimeUnit.HOURS.toMillis(1));

// enable term statistics, same as in SAITester
CassandraRelevantProperties.SAI_QUERY_OPTIMIZATION_USE_TERM_STATISTICS.setBoolean(true);

try (Cluster cluster = init(Cluster.build(1)
.withConfig(c -> c.set("slow_query_log_timeout_in_ms", SLOW_QUERY_LOG_TIMEOUT_IN_MS))
.withInstanceInitializer(BB::install)
Expand Down
33 changes: 21 additions & 12 deletions test/unit/org/apache/cassandra/db/MultiRangeReadCommandCQLTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.sai.SAIUtil;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.utils.Pair;
import org.assertj.core.api.Assertions;

Expand Down Expand Up @@ -93,18 +95,25 @@ public void testToCQLString()
multiRangeError);

// test ANN index-based ORDER BY
createTable("CREATE TABLE %s (k int, c int, n int, v vector<float, 2>, PRIMARY KEY (k, c))");
createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'");
String truncationError = "no viable alternative at input '..'";
assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT * FROM %s WHERE (token(k) <= -1 OR token(k) > -1) ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT * FROM %s WHERE (token(k) <= ? OR token(k) > ?) ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT * FROM %s WHERE (token(k) <= -1 OR token(k) > -1) AND n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT * FROM %s WHERE (token(k) <= ? OR token(k) > ?) AND n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
SAIUtil.withVersionsOnOrAfter(Version.JVECTOR_EARLIEST, version -> {

createTable("CREATE TABLE %s (k int, c int, n int, v vector<float, 2>, PRIMARY KEY (k, c))");
createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'");

// the synthetic score column makes us not to send a wildcard column filter but individual column selections
String columns = version.onOrAfter(Version.SYNTHETIC_COLUMN_EARLIEST) ? "n, v" : "*";
String truncationError = "no viable alternative at input '..'";

assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT " + columns + " FROM %s WHERE (token(k) <= -1 OR token(k) > -1) ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT " + columns + " FROM %s WHERE (token(k) <= ? OR token(k) > ?) ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT " + columns + " FROM %s WHERE (token(k) <= -1 OR token(k) > -1) AND n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT " + columns + " FROM %s WHERE (token(k) <= ? OR token(k) > ?) AND n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
});
}

@Override
Expand Down
33 changes: 21 additions & 12 deletions test/unit/org/apache/cassandra/db/PartitionRangeReadCommandCQLTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.index.StubIndex;
import org.apache.cassandra.index.sai.SAIUtil;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.assertj.core.api.Assertions;
Expand Down Expand Up @@ -130,18 +132,25 @@ public void testToCQLString()
"SELECT * FROM %s WHERE n = ? ORDER BY n ASC LIMIT 10 ALLOW FILTERING");

// test ANN index-based ORDER BY
createTable("CREATE TABLE %s (k int, c int, n int, v vector<float, 2>, PRIMARY KEY (k, c))");
createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'");
String truncationError = "no viable alternative at input '..'";
assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT * FROM %s ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT * FROM %s ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT * FROM %s WHERE n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
SAIUtil.withVersionsOnOrAfter(Version.JVECTOR_EARLIEST, version -> {

createTable("CREATE TABLE %s (k int, c int, n int, v vector<float, 2>, PRIMARY KEY (k, c))");
createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'");

// the synthetic score column makes us not to send a wildcard column filter but individual column selections
String columns = version.onOrAfter(Version.SYNTHETIC_COLUMN_EARLIEST) ? "n, v" : "*";
String truncationError = "no viable alternative at input '..'";

assertToCQLString("SELECT * FROM %s ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT " + columns + " FROM %s ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT " + columns + " FROM %s ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
assertToCQLString("SELECT * FROM %s WHERE n = 0 ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT " + columns + " FROM %s WHERE n = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT " + columns + " FROM %s WHERE n = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
truncationError);
});

// test literals
createTable("CREATE TABLE %s (k text, c text, m map<text, text>, PRIMARY KEY (k, c))");
Expand Down
23 changes: 16 additions & 7 deletions test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.junit.Test;

import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.index.sai.SAIUtil;
import org.apache.cassandra.index.sai.disk.format.Version;

import static java.util.Arrays.asList;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -208,13 +210,20 @@ public void testToCQLString()
"SELECT * FROM %s WHERE k = ? AND c IN (?, ?) ORDER BY n ASC LIMIT 10 ALLOW FILTERING");

// test ANN index-based ORDER BY
createTable("CREATE TABLE %s (k int, c int, n int, v vector<float, 2>, PRIMARY KEY (k, c))");
createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'");
assertToCQLString("SELECT * FROM %s WHERE k = 0 ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT * FROM %s WHERE k = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT * FROM %s WHERE k = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
"no viable alternative at input '..'");
SAIUtil.withVersionsOnOrAfter(Version.JVECTOR_EARLIEST, version -> {

createTable("CREATE TABLE %s (k int, c int, n int, v vector<float, 2>, PRIMARY KEY (k, c))");
createIndex("CREATE CUSTOM INDEX ON %s(v) USING 'StorageAttachedIndex'");
createIndex("CREATE CUSTOM INDEX ON %s(n) USING 'StorageAttachedIndex'");

// the synthetic score column makes us not to send a wildcard column filter but individual column selections
String columns = version.onOrAfter(Version.SYNTHETIC_COLUMN_EARLIEST) ? "n, v" : "*";

assertToCQLString("SELECT * FROM %s WHERE k = 0 ORDER BY v ANN OF [1, 2] LIMIT 10",
"SELECT " + columns + " FROM %s WHERE k = 0 ORDER BY v ANN OF [1.0, ... LIMIT 10 ALLOW FILTERING",
"SELECT " + columns + " FROM %s WHERE k = ? ORDER BY v ANN OF ? LIMIT 10 ALLOW FILTERING",
"no viable alternative at input '..'");
});

// test with index and multi-column clustering
createTable("CREATE TABLE %s (k int, c1 int, c2 int,v int, PRIMARY KEY (k, c1, c2))");
Expand Down
27 changes: 27 additions & 0 deletions test/unit/org/apache/cassandra/index/sai/SAIUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.Map;
import java.util.function.Consumer;

import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.cql3.CQLTester;
Expand Down Expand Up @@ -136,6 +137,32 @@ public static void setEnableFused(boolean enableFused)
}
}

/**
* Runs the provided tester for each SAI version on or after the provided version.
*
* @param version the version to start testing from
* @param tester a tester consuming the tested version
*/
public static void withVersionsOnOrAfter(Version version, Consumer<Version> tester)
{
Version currentVersion = currentVersion();
try
{
for (Version testedVersion : Version.ALL)
{
if (!testedVersion.onOrAfter(version))
continue;

SAIUtil.setCurrentVersion(testedVersion);
tester.accept(testedVersion);
}
}
finally
{
SAIUtil.setCurrentVersion(currentVersion);
}
}

public static class CustomVersionSelector implements Version.Selector
{
private final Version defaultVersion;
Expand Down
Loading