Skip to content

CASSANDRA-21157 add table_id to system_distributed.compression_dictionaries table #4601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smiklosovic wants to merge 2 commits into
base: trunk
Choose a base branch
from
Open

CASSANDRA-21157 add table_id to system_distributed.compression_dictionaries table #4601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
967aaa0
add table_id to system_distributed.compression_dictionaries table
smiklosovic Feb 4, 2026
773db9a
clear orphaned
smiklosovic Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions src/java/org/apache/cassandra/db/compression/CompressionDictionary.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,11 +261,13 @@ static LightweightCompressionDictionary createFromRowLightweight(UntypedResultSe
int size = row.getInt("dict_length");
String keyspaceName = row.getString("keyspace_name");
String tableName = row.getString("table_name");
String tableId = row.getString("table_id");

try
{
return new LightweightCompressionDictionary(keyspaceName,
tableName,
tableId,
new DictId(CompressionDictionary.Kind.valueOf(kindStr), dictId),
checksum,
size);
Expand Down Expand Up @@ -430,21 +432,37 @@ class LightweightCompressionDictionary
{
public final String keyspaceName;
public final String tableName;
public final String tableId;
public final DictId dictId;
public final int checksum;
public final int size;

public LightweightCompressionDictionary(String keyspaceName,
String tableName,
String tableId,
DictId dictId,
int checksum,
int size)
{
this.keyspaceName = keyspaceName;
this.tableName = tableName;
this.tableId = tableId;
this.dictId = dictId;
this.checksum = checksum;
this.size = size;
}

@Override
public String toString()
{
return "LightweightCompressionDictionary{" +
"keyspaceName='" + keyspaceName + '\'' +
", tableName='" + tableName + '\'' +
", tableId='" + tableId + '\'' +
", dictId=" + dictId +
", checksum=" + checksum +
", size=" + size +
'}';
}
}
}
28 changes: 25 additions & 3 deletions src/java/org/apache/cassandra/db/compression/CompressionDictionaryDetailsTabularData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,25 @@

public class CompressionDictionaryDetailsTabularData
{
/**
* Position inside index names of tabular type of tabular data returned upon
* listing dictionaries where table id is expected to be located.
* We do not need to process this entry at all time, e.g. when not listing
* orphaned compression dictionaries.
*/
public static final int TABULAR_DATA_TYPE_TABLE_ID_INDEX = 2;

/**
* Position inside index names of tabular type of tabular data returned upon
* listing dictionaries where raw dictionary is expected to be located.
* We do not need to process this entry as listing does not contain any raw dictionary,
* only exporting does.
*/
public static final int TABULAR_DATA_TYPE_RAW_DICTIONARY_INDEX = 3;
public static final int TABULAR_DATA_TYPE_RAW_DICTIONARY_INDEX = 4;

public static final String KEYSPACE_NAME = "Keyspace";
public static final String TABLE_NAME = "Table";
public static final String TABLE_ID_NAME = "TableId";
public static final String DICT_ID_NAME = "DictId";
public static final String DICT_NAME = "Dict";
public static final String KIND_NAME = "Kind";
Expand All @@ -58,6 +67,7 @@ public class CompressionDictionaryDetailsTabularData

private static final String[] ITEM_NAMES = new String[]{ KEYSPACE_NAME,
TABLE_NAME,
TABLE_ID_NAME,
DICT_ID_NAME,
DICT_NAME,
KIND_NAME,
Expand All @@ -66,6 +76,7 @@ public class CompressionDictionaryDetailsTabularData

private static final String[] ITEM_DESCS = new String[]{ "keyspace",
"table",
"table_id",
"dictionary_id",
"dictionary_bytes",
"kind",
Expand All @@ -84,6 +95,7 @@ public class CompressionDictionaryDetailsTabularData
{
ITEM_TYPES = new OpenType[]{ SimpleType.STRING, // keyspace
SimpleType.STRING, // table
SimpleType.STRING, // tableId
SimpleType.LONG, // dict id
new ArrayType<String[]>(SimpleType.BYTE, true), // dict bytes
SimpleType.STRING, // kind
Expand Down Expand Up @@ -115,6 +127,7 @@ public static CompositeData fromLightweightCompressionDictionary(LightweightComp
{
dictionary.keyspaceName,
dictionary.tableName,
dictionary.tableId,
dictionary.dictId.id,
null, // on purpose not returning actual dictionary
dictionary.dictId.kind.name(),
Expand All @@ -133,10 +146,11 @@ public static CompositeData fromLightweightCompressionDictionary(LightweightComp
*
* @param keyspace keyspace of a dictionary
* @param table table of a dictionary
* @param tableId id of a table dictionary is for
* @param dictionary dictionary itself
* @return composite data representing dictionary
*/
public static CompositeData fromCompressionDictionary(String keyspace, String table, CompressionDictionary dictionary)
public static CompositeData fromCompressionDictionary(String keyspace, String table, String tableId, CompressionDictionary dictionary)
{
try
{
Expand All @@ -146,6 +160,7 @@ public static CompositeData fromCompressionDictionary(String keyspace, String ta
{
keyspace,
table,
tableId,
dictionary.dictId().id,
dictionary.rawDictionary(),
dictionary.kind().name(),
Expand Down Expand Up @@ -176,6 +191,7 @@ public static CompositeData fromCompressionDictionaryDataObject(CompressionDicti
{
dataObject.keyspace,
dataObject.table,
dataObject.tableId,
dataObject.dictId,
dataObject.dict,
dataObject.kind,
Expand All @@ -200,6 +216,7 @@ public static CompressionDictionaryDataObject fromCompositeData(CompositeData co
{
return new CompressionDictionaryDataObject((String) compositeData.get(CompressionDictionaryDetailsTabularData.KEYSPACE_NAME),
(String) compositeData.get(CompressionDictionaryDetailsTabularData.TABLE_NAME),
(String) compositeData.get(CompressionDictionaryDetailsTabularData.TABLE_ID_NAME),
(Long) compositeData.get(CompressionDictionaryDetailsTabularData.DICT_ID_NAME),
(byte[]) compositeData.get(CompressionDictionaryDetailsTabularData.DICT_NAME),
(String) compositeData.get(CompressionDictionaryDetailsTabularData.KIND_NAME),
Expand All @@ -211,6 +228,7 @@ public static class CompressionDictionaryDataObject
{
public final String keyspace;
public final String table;
public final String tableId;
public final long dictId;
public final byte[] dict;
public final String kind;
Expand All @@ -220,6 +238,7 @@ public static class CompressionDictionaryDataObject
@JsonCreator
public CompressionDictionaryDataObject(@JsonProperty("keyspace") String keyspace,
@JsonProperty("table") String table,
@JsonProperty("tableId") String tableId,
@JsonProperty("dictId") long dictId,
@JsonProperty("dict") byte[] dict,
@JsonProperty("kind") String kind,
Expand All @@ -228,6 +247,7 @@ public CompressionDictionaryDataObject(@JsonProperty("keyspace") String keyspace
{
this.keyspace = keyspace;
this.table = table;
this.tableId = tableId;
this.dictId = dictId;
this.dict = dict;
this.kind = kind;
Expand All @@ -241,7 +261,7 @@ public CompressionDictionaryDataObject(@JsonProperty("keyspace") String keyspace
* An object of this class is considered to be valid if:
*
* <ul>
* <li>keyspace and table are not null</li>
* <li>keyspace, table and table id are not null</li>
* <li>dict id is lower than 0</li>
* <li>dict is not null nor empty</li>
* <li>dict length is less than or equal to 1MiB</li>
Expand All @@ -257,6 +277,8 @@ private void validate()
throw new IllegalArgumentException("Keyspace not specified.");
if (table == null)
throw new IllegalArgumentException("Table not specified.");
if (tableId == null)
throw new IllegalArgumentException("Table id not specified");
if (dictId <= 0)
throw new IllegalArgumentException("Provided dictionary id must be positive but it is '" + dictId + "'.");
if (dict == null || dict.length == 0)
Expand Down
2 changes: 1 addition & 1 deletion src/java/org/apache/cassandra/db/compression/CompressionDictionaryEventHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void onNewDictionaryAvailable(CompressionDictionary.DictId dictionaryId)
return;
}

CompressionDictionary dictionary = SystemDistributedKeyspace.retrieveCompressionDictionary(keyspaceName, tableName, dictionaryId.id);
CompressionDictionary dictionary = SystemDistributedKeyspace.retrieveCompressionDictionary(keyspaceName, tableName, cfs.metadata().id.toLongString(), dictionaryId.id);
cache.add(dictionary);
}
catch (Exception e)
Expand Down
18 changes: 10 additions & 8 deletions src/java/org/apache/cassandra/db/compression/CompressionDictionaryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage

private final String keyspaceName;
private final String tableName;
private final String tableId;
private final ColumnFamilyStore columnFamilyStore;
private volatile boolean mbeanRegistered;
private volatile boolean isEnabled;
Expand All @@ -73,12 +74,13 @@ public CompressionDictionaryManager(ColumnFamilyStore columnFamilyStore, boolean
{
this.keyspaceName = columnFamilyStore.keyspace.getName();
this.tableName = columnFamilyStore.getTableName();
this.tableId = columnFamilyStore.metadata().id.toLongString();
this.columnFamilyStore = columnFamilyStore;

this.isEnabled = columnFamilyStore.metadata().params.compression.isDictionaryCompressionEnabled();
this.cache = new CompressionDictionaryCache();
this.eventHandler = new CompressionDictionaryEventHandler(columnFamilyStore, cache);
this.scheduler = new CompressionDictionaryScheduler(keyspaceName, tableName, cache, isEnabled);
this.scheduler = new CompressionDictionaryScheduler(keyspaceName, tableName, tableId, cache, isEnabled);
if (isEnabled)
{
// Initialize components
Expand Down Expand Up @@ -270,7 +272,7 @@ public CompositeData getTrainingState()
@Override
public TabularData listCompressionDictionaries()
{
List<LightweightCompressionDictionary> dictionaries = SystemDistributedKeyspace.retrieveLightweightCompressionDictionaries(keyspaceName, tableName);
List<LightweightCompressionDictionary> dictionaries = SystemDistributedKeyspace.retrieveLightweightCompressionDictionaries(keyspaceName, tableName, tableId);
TabularDataSupport tableData = new TabularDataSupport(CompressionDictionaryDetailsTabularData.TABULAR_TYPE);

if (dictionaries == null)
Expand All @@ -289,21 +291,21 @@ public TabularData listCompressionDictionaries()
@Override
public CompositeData getCompressionDictionary()
{
CompressionDictionary compressionDictionary = SystemDistributedKeyspace.retrieveLatestCompressionDictionary(keyspaceName, tableName);
CompressionDictionary compressionDictionary = SystemDistributedKeyspace.retrieveLatestCompressionDictionary(keyspaceName, tableName, tableId);
if (compressionDictionary == null)
return null;

return CompressionDictionaryDetailsTabularData.fromCompressionDictionary(keyspaceName, tableName, compressionDictionary);
return CompressionDictionaryDetailsTabularData.fromCompressionDictionary(keyspaceName, tableName, tableId, compressionDictionary);
}

@Override
public CompositeData getCompressionDictionary(long dictId)
{
CompressionDictionary compressionDictionary = SystemDistributedKeyspace.retrieveCompressionDictionary(keyspaceName, tableName, dictId);
CompressionDictionary compressionDictionary = SystemDistributedKeyspace.retrieveCompressionDictionary(keyspaceName, tableName, tableId, dictId);
if (compressionDictionary == null)
return null;

return CompressionDictionaryDetailsTabularData.fromCompressionDictionary(keyspaceName, tableName, compressionDictionary);
return CompressionDictionaryDetailsTabularData.fromCompressionDictionary(keyspaceName, tableName, tableId, compressionDictionary);
}

@Override
Expand Down Expand Up @@ -333,7 +335,7 @@ public synchronized void importCompressionDictionary(CompositeData compositeData

CompressionDictionary.DictId dictId = new CompressionDictionary.DictId(kind, dataObject.dictId);

LightweightCompressionDictionary latestCompressionDictionary = SystemDistributedKeyspace.retrieveLightweightLatestCompressionDictionary(keyspaceName, tableName);
LightweightCompressionDictionary latestCompressionDictionary = SystemDistributedKeyspace.retrieveLightweightLatestCompressionDictionary(keyspaceName, tableName, tableId);
if (latestCompressionDictionary != null && latestCompressionDictionary.dictId.id > dictId.id)
{
throw new IllegalArgumentException(format("Dictionary to import has older dictionary id (%s) than the latest compression dictionary (%s) for table %s.%s",
Expand Down Expand Up @@ -438,7 +440,7 @@ private void storeDictionary(CompressionDictionary dictionary)
return;
}

SystemDistributedKeyspace.storeCompressionDictionary(keyspaceName, tableName, dictionary);
SystemDistributedKeyspace.storeCompressionDictionary(keyspaceName, tableName, tableId, dictionary);
cache.add(dictionary);
}

Expand Down
5 changes: 4 additions & 1 deletion src/java/org/apache/cassandra/db/compression/CompressionDictionaryScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch

private final String keyspaceName;
private final String tableName;
private final String tableId;
private final ICompressionDictionaryCache cache;
private final AtomicBoolean manualTrainingInProgress = new AtomicBoolean(false);

Expand All @@ -59,11 +60,13 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch

public CompressionDictionaryScheduler(String keyspaceName,
String tableName,
String tableId,
ICompressionDictionaryCache cache,
boolean isEnabled)
{
this.keyspaceName = keyspaceName;
this.tableName = tableName;
this.tableId = tableId;
this.cache = cache;
this.isEnabled = isEnabled;
}
Expand Down Expand Up @@ -135,7 +138,7 @@ private void refreshDictionaryFromSystemTable()
return;
}

CompressionDictionary dictionary = SystemDistributedKeyspace.retrieveLatestCompressionDictionary(keyspaceName, tableName);
CompressionDictionary dictionary = SystemDistributedKeyspace.retrieveLatestCompressionDictionary(keyspaceName, tableName, tableId);
cache.add(dictionary);
}
catch (Exception e)
Expand Down
Loading