natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rework ZSTD dictionary compression logic to create a trainer per training 

patch by Stefan Miklosovic; reviewed by Yifan Cai for CASSANDRA-21209
This commit is contained in:
Stefan Miklosovic 2026-03-11 16:35:02 +01:00
parent 922c8fb9c9
commit a54d227c86
No known key found for this signature in database
GPG Key ID: 32F35CB2F546D93E
16 changed files with 362 additions and 433 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
6.0-alpha1
* Rework ZSTD dictionary compression logic to create a trainer per training (CASSANDRA-21209)
* Improve performance when calculating settled placements during range movements (CASSANDRA-21144)
* Make shadow gossip round parameters configurable for testing (CASSANDRA-21149)
* Avoid potential gossip thread deadlock during decommission (CASSANDRA-21143)

210
src/java/org/apache/cassandra/db/compression/CompressionDictionaryManager.java
View File

@ -22,18 +22,16 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.TabularData;
import javax.management.openmbean.TabularDataSupport;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compression.CompressionDictionary.LightweightCompressionDictionary;
@ -47,12 +45,6 @@ import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.MBeanWrapper.OnException;
import static java.lang.String.format;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.DEFAULT_TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_VALUE;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.DEFAULT_TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_VALUE;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.DEFAULT_TRAINING_MIN_FREQUENCY;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.TRAINING_MIN_FREQUENCY_PARAMETER_NAME;
import static org.apache.cassandra.schema.SystemDistributedKeyspace.retrieveLightweightLatestCompressionDictionary;
public class CompressionDictionaryManager implements CompressionDictionaryManagerMBean,
@ -68,12 +60,13 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
private final ColumnFamilyStore columnFamilyStore;
private volatile boolean mbeanRegistered;
private volatile boolean isEnabled;
private volatile CompressionDictionary.Kind kind;
private volatile CompressionParams compressionParams;
// Components
private final ICompressionDictionaryEventHandler eventHandler;
private final ICompressionDictionaryCache cache;
private final ICompressionDictionaryScheduler scheduler;
private ICompressionDictionaryTrainer trainer = null;
public CompressionDictionaryManager(ColumnFamilyStore columnFamilyStore, boolean registerBookkeeping)
{
@ -82,20 +75,15 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
this.tableId = columnFamilyStore.metadata().id.toLongString();
this.columnFamilyStore = columnFamilyStore;
this.isEnabled = columnFamilyStore.metadata().params.compression.isDictionaryCompressionEnabled();
this.compressionParams = columnFamilyStore.metadata().params.compression;
this.isEnabled = this.compressionParams.isDictionaryCompressionEnabled();
this.kind = columnFamilyStore.metadata().params.compression.getCompressionDictionaryKind();
this.cache = new CompressionDictionaryCache();
this.eventHandler = new CompressionDictionaryEventHandler(columnFamilyStore, cache);
this.scheduler = new CompressionDictionaryScheduler(keyspaceName, tableName, tableId, cache, isEnabled);
if (isEnabled)
{
// Initialize components
this.trainer = ICompressionDictionaryTrainer.create(keyspaceName, tableName,
columnFamilyStore.metadata().params.compression);
trainer.setDictionaryTrainedListener(this::handleNewDictionary);
scheduler.scheduleRefreshTask();
trainer.start(createTrainingConfig());
}
if (registerBookkeeping && isEnabled)
@ -123,40 +111,14 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
*/
public synchronized void maybeReloadFromSchema(CompressionParams newParams)
{
this.isEnabled = newParams.isDictionaryCompressionEnabled();
this.compressionParams = newParams;
this.isEnabled = compressionParams.isDictionaryCompressionEnabled();
this.kind = compressionParams.getCompressionDictionaryKind();
scheduler.setEnabled(isEnabled);
if (isEnabled)
{
registerMbean();
// Check if we need a new trainer due to compression parameter changes
boolean needsNewTrainer = shouldCreateNewTrainer(newParams);
if (needsNewTrainer)
{
// Close existing trainer and create a new one
if (trainer != null)
{
try
{
trainer.close();
}
catch (Exception e)
{
logger.warn("Failed to close existing trainer for {}.{}", keyspaceName, tableName, e);
}
}
trainer = ICompressionDictionaryTrainer.create(keyspaceName, tableName, newParams);
trainer.setDictionaryTrainedListener(this::handleNewDictionary);
}
scheduler.scheduleRefreshTask();
// Start trainer if it exists
if (trainer != null)
{
trainer.start(createTrainingConfig());
}
return;
}
@ -218,11 +180,6 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
throw new UnsupportedOperationException("Table " + keyspaceName + '.' + tableName + " does not support dictionary compression");
}
if (trainer == null)
{
throw new IllegalStateException("Dictionary trainer is not available for table " + keyspaceName + '.' + tableName);
}
// resolve training config and fail fast when invalid, so we do not reach logic which would e.g. flush unnecessarily.
CompressionDictionaryTrainingConfig trainingConfig = createTrainingConfig(parameters);
@ -230,7 +187,7 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
columnFamilyStore.getTableName(),
columnFamilyStore.metadata.id.toLongString());
checkTrainingFrequency(dictionary);
checkTrainingFrequency(dictionary, trainingConfig);
// SSTable-based training: sample from existing SSTables
@ -241,8 +198,7 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
if (refViewFragment.sstables.isEmpty())
{
logger.info("No SSTables available for training in table {}.{}, flushing memtable first",
keyspaceName, tableName);
logger.info("No SSTables available for training in table {}.{}, flushing memtable first", keyspaceName, tableName);
columnFamilyStore.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
refViewFragment = columnFamilyStore.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
@ -253,22 +209,17 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
}
}
logger.info("Starting SSTable-based training for {}.{} with {} SSTables",
keyspaceName, tableName, refViewFragment.sstables.size());
trainer.start(trainingConfig);
scheduler.scheduleSSTableBasedTraining(trainer, refViewFragment, trainingConfig, force);
scheduler.scheduleSSTableBasedTraining(refViewFragment,
compressionParams,
trainingConfig,
this::handleNewDictionary,
force);
}
@Override
public CompositeData getTrainingState()
{
ICompressionDictionaryTrainer dictionaryTrainer = trainer;
if (dictionaryTrainer == null)
{
return TrainingState.notStarted().toCompositeData();
}
return dictionaryTrainer.getTrainingState().toCompositeData();
return scheduler.getLastTrainingState().toCompositeData();
}
@Override
@ -313,7 +264,7 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
@Override
public synchronized void importCompressionDictionary(CompositeData compositeData)
{
if (!isEnabled)
if (!isEnabled || this.kind == null)
{
throw new IllegalStateException(format("The compression on table %s.%s is not enabled or SSTable compressor is not a dictionary compressor.",
keyspaceName, tableName));
@ -328,11 +279,11 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
CompressionDictionary.Kind kind = CompressionDictionary.Kind.valueOf(dataObject.kind);
if (trainer.kind() != kind)
if (this.kind != kind)
{
throw new IllegalArgumentException(format("It is not possible to import compression dictionaries of kind " +
"%s into table %s.%s which supports compression dictionaries of kind %s.",
kind, keyspaceName, tableName, trainer.kind()));
kind, keyspaceName, tableName, this.kind));
}
CompressionDictionary.DictId dictId = new CompressionDictionary.DictId(kind, dataObject.dictId);
@ -346,7 +297,7 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
dictId.id, latestCompressionDictionary.dictId.id, keyspaceName, tableName));
}
checkTrainingFrequency(latestCompressionDictionary);
checkTrainingFrequency(latestCompressionDictionary, createTrainingConfig(Map.of()));
}
handleNewDictionary(kind.createDictionary(dictId, dataObject.dict, dataObject.dictChecksum));
@ -359,30 +310,17 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
public synchronized void close()
{
unregisterMbean();
if (trainer != null)
{
closeQuitely(trainer, "CompressionDictionaryTrainer");
trainer = null;
}
closeQuitely(cache, "CompressionDictionaryCache");
closeQuitely(scheduler, "CompressionDictionaryScheduler");
}
private void handleNewDictionary(CompressionDictionary dictionary)
void handleNewDictionary(CompressionDictionary dictionary)
{
// sequence meatters; persist the new dictionary before broadcasting to others.
storeDictionary(dictionary);
onNewDictionaryTrained(dictionary.dictId());
}
/**
* @return training configuration with max dictionary size and total sample size from CQL table compression params.
*/
private CompressionDictionaryTrainingConfig createTrainingConfig()
{
return createTrainingConfig(Map.of());
}
/**
* Returns configuration for training where max dictionary size and total sample size can be supplied by a
* user, e.g. upon the invocation of training method via JMX.
@ -393,92 +331,37 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
*/
private CompressionDictionaryTrainingConfig createTrainingConfig(Map<String, String> parameters)
{
CompressionParams compressionParams = columnFamilyStore.metadata().params.compression;
return CompressionDictionaryTrainingConfig
.builder()
.maxDictionarySize(getCompressionDictionaryTrainingMaxDictionarySize(compressionParams, parameters))
.maxTotalSampleSize(getCompressionDictionaryTrainingMaxTotalSampleSize(compressionParams, parameters))
.maxDictionarySize(CompressionDictionaryTrainingConfig.getMaxDictionarySizeWithUserSuppliedParams(compressionParams, parameters))
.maxTotalSampleSize(CompressionDictionaryTrainingConfig.getMaxTotalSampleSizeWithUserSuppliedParams(compressionParams, parameters))
.minTrainingFrequency(CompressionDictionaryTrainingConfig.getMinTrainingFrequency(compressionParams.getOtherOptions()))
.chunkSize(compressionParams.chunkLength())
.build();
}
private int getCompressionDictionaryTrainingMaxDictionarySize(CompressionParams compressionParams, Map<String, String> parameters)
{
return internalTrainingParameterResolution(compressionParams,
parameters.get(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME),
TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_VALUE);
}
private int getCompressionDictionaryTrainingMaxTotalSampleSize(CompressionParams compressionParams, Map<String, String> parameters)
{
return internalTrainingParameterResolution(compressionParams,
parameters.get(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME),
TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_VALUE);
}
private DurationSpec.IntMinutesBound getCompressionDictionaryMinTrainingFrequency(CompressionParams compressionParams)
{
String resolvedValue = compressionParams.getOtherOptions().getOrDefault(TRAINING_MIN_FREQUENCY_PARAMETER_NAME, DEFAULT_TRAINING_MIN_FREQUENCY);
try
{
return new DurationSpec.IntMinutesBound(resolvedValue);
}
catch (Throwable t)
{
throw new IllegalArgumentException(String.format("Invalid value for %s: %s. Reason: %s",
TRAINING_MIN_FREQUENCY_PARAMETER_NAME,
resolvedValue,
t.getMessage()));
}
}
private void checkTrainingFrequency(LightweightCompressionDictionary lastDictionary)
private void checkTrainingFrequency(LightweightCompressionDictionary lastDictionary, CompressionDictionaryTrainingConfig config)
{
Instant lastTraining = lastDictionary == null ? null : lastDictionary.createdAt;
DurationSpec.IntMinutesBound minTrainingFrequency = getCompressionDictionaryMinTrainingFrequency(columnFamilyStore.metadata().params.compression);
// if there is no dictionary trained so far or min frequency is 0 - that is we can train as often as we want -
// then do not check if we can
if (lastTraining != null && minTrainingFrequency.toMinutes() != 0)
if (lastTraining != null && config.minTrainingFrequency != 0)
{
Instant now = FBUtilities.now();
int minTrainingFrequencyMinutes = minTrainingFrequency.toMinutes();
if (lastTraining.isAfter(now.minus(minTrainingFrequencyMinutes, ChronoUnit.MINUTES)))
if (lastTraining.isAfter(now.minus(config.minTrainingFrequency, ChronoUnit.MINUTES)))
{
Instant nextEarliestTraining = lastTraining.plus(minTrainingFrequencyMinutes, ChronoUnit.MINUTES);
Instant nextEarliestTraining = lastTraining.plus(config.minTrainingFrequency, ChronoUnit.MINUTES);
throw new IllegalArgumentException(format("The next training or importing can occur only at least after %s from the last training which happened at %s. " +
"You can train again no earlier than at %s.",
minTrainingFrequency,
new DurationSpec.IntMinutesBound(config.minTrainingFrequency, TimeUnit.MINUTES),
lastTraining,
nextEarliestTraining));
}
}
}
private int internalTrainingParameterResolution(CompressionParams compressionParams,
String userSuppliedValue,
String parameterName,
String defaultParameterValue)
{
String resolvedValue = null;
try
{
if (userSuppliedValue == null)
resolvedValue = compressionParams.getOtherOptions().getOrDefault(parameterName, defaultParameterValue);
else
resolvedValue = userSuppliedValue;
return new DataStorageSpec.IntBytesBound(resolvedValue).toBytes();
}
catch (Throwable t)
{
throw new IllegalArgumentException(String.format("Invalid value for %s: %s", parameterName, resolvedValue));
}
}
private void storeDictionary(CompressionDictionary dictionary)
{
if (!isEnabled)
@ -490,27 +373,6 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
cache.add(dictionary);
}
/**
* Determines if a new trainer should be created based on compression parameter changes.
* A new trainer is needed when no existing trainer exists or when the existing trainer
* is not compatible with the new compression parameters.
* <p>
* The method is (and should be) only invoked inside {@link #maybeReloadFromSchema(CompressionParams)},
* which is guarded by synchronized.
*
* @param newParams the new compression parameters
* @return true if a new trainer should be created
*/
private boolean shouldCreateNewTrainer(CompressionParams newParams)
{
if (trainer == null)
{
return true;
}
return !trainer.isCompatibleWith(newParams);
}
private void registerMbean()
{
if (!mbeanRegistered)
@ -540,16 +402,4 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
logger.warn("Failed closing {}", objectName, exception);
}
}
@VisibleForTesting
boolean isReady()
{
return trainer != null && trainer.isReady();
}
@VisibleForTesting
ICompressionDictionaryTrainer trainer()
{
return trainer;
}
}

96
src/java/org/apache/cassandra/db/compression/CompressionDictionaryScheduler.java
View File

@ -21,6 +21,8 @@ package org.apache.cassandra.db.compression;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import com.google.common.annotations.VisibleForTesting;
@ -30,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.schema.SystemDistributedKeyspace;
/**
@ -48,7 +51,9 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch
private final String tableName;
private final String tableId;
private final ICompressionDictionaryCache cache;
private final AtomicBoolean manualTrainingInProgress = new AtomicBoolean(false);
private final AtomicBoolean trainingInProgress = new AtomicBoolean(false);
private final AtomicReference<TrainingState> lastTrainingState = new AtomicReference<>(TrainingState.notStarted());
private volatile ICompressionDictionaryTrainer activeTrainer;
private volatile ScheduledFuture<?> scheduledRefreshTask;
private volatile boolean isEnabled;
@ -83,31 +88,59 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch
}
@Override
public void scheduleSSTableBasedTraining(ICompressionDictionaryTrainer trainer,
ColumnFamilyStore.RefViewFragment refViewFragment,
public void scheduleSSTableBasedTraining(ColumnFamilyStore.RefViewFragment refViewFragment,
CompressionParams compressionParams,
CompressionDictionaryTrainingConfig config,
Consumer<CompressionDictionary> listener,
boolean force)
{
if (!manualTrainingInProgress.compareAndSet(false, true))
if (!trainingInProgress.compareAndSet(false, true))
{
refViewFragment.close();
throw new IllegalStateException("Training already in progress for table " + keyspaceName + '.' + tableName);
}
logger.info("Starting SSTable-based dictionary training for {}.{} from {} SSTables",
keyspaceName, tableName, refViewFragment.sstables.size());
ICompressionDictionaryTrainer trainer;
// Run the SSTableSamplingTask asynchronously
SSTableSamplingTask task = new SSTableSamplingTask(refViewFragment, trainer, config, force);
ScheduledExecutors.nonPeriodicTasks.submit(task);
try
{
trainer = ICompressionDictionaryTrainer.create(keyspaceName, tableName, compressionParams);
trainer.setDictionaryTrainedListener(listener);
}
catch (Throwable t)
{
trainingInProgress.set(false);
refViewFragment.close();
throw t;
}
if (trainer.start(config))
{
activeTrainer = trainer;
lastTrainingState.set(trainer.getTrainingState());
logger.info("Starting SSTable-based dictionary training for {}.{} from {} SSTables",
keyspaceName, tableName, refViewFragment.sstables.size());
SSTableSamplingTask task = new SSTableSamplingTask(refViewFragment, trainer, config, force);
// trainer is eventually closed here, as well as indicating
// in manualTrainingInProgress that it was finished
ScheduledExecutors.nonPeriodicTasks.submit(task);
}
else
{
finishTraining(trainer.getTrainingState());
cleanup(refViewFragment, trainer);
}
}
/**
* Cancels the in-progress manual training task.
*/
private void cancelManualTraining()
private void finishTraining(TrainingState trainingState)
{
manualTrainingInProgress.compareAndSet(true, false);
lastTrainingState.set(trainingState);
activeTrainer = null;
trainingInProgress.compareAndSet(true, false);
}
/**
@ -121,6 +154,15 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch
this.isEnabled = enabled;
}
@Override
public TrainingState getLastTrainingState()
{
ICompressionDictionaryTrainer trainer = activeTrainer;
if (trainer != null)
return trainer.getTrainingState();
return lastTrainingState.get();
}
/**
* Refreshes dictionary from system table and updates the cache.
* This method is called periodically by the scheduled refresh task.
@ -153,7 +195,7 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch
scheduledRefreshTask = null;
}
cancelManualTraining();
finishTraining(TrainingState.notStarted());
}
/**
@ -195,7 +237,6 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch
// Use the force parameter from the task
trainer.trainDictionaryAsync(force)
.addCallback((dictionary, throwable) -> {
cancelManualTraining();
if (throwable != null)
{
logger.error("SSTable-based dictionary training failed for {}.{}: {}",
@ -206,23 +247,36 @@ public class CompressionDictionaryScheduler implements ICompressionDictionarySch
logger.info("SSTable-based dictionary training completed for {}.{}",
keyspaceName, tableName);
}
finishTraining(trainer.getTrainingState());
cleanup(refViewFragment, trainer);
});
}
catch (Exception e)
{
logger.error("Failed to sample from SSTables for {}.{}", keyspaceName, tableName, e);
cancelManualTraining();
}
finally
{
refViewFragment.close();
finishTraining(trainer.getTrainingState());
cleanup(refViewFragment, trainer);
}
}
}
@VisibleForTesting
boolean isManualTrainingRunning()
private void cleanup(ColumnFamilyStore.RefViewFragment refViewFragment, ICompressionDictionaryTrainer trainer)
{
return manualTrainingInProgress.get();
try
{
trainer.close();
}
catch (Throwable t)
{
logger.debug("Unable to close trainer.", t);
}
refViewFragment.close();
}
@VisibleForTesting
boolean isTrainingRunning()
{
return trainingInProgress.get();
}
}

127
src/java/org/apache/cassandra/db/compression/CompressionDictionaryTrainingConfig.java
View File

@ -18,8 +18,23 @@
package org.apache.cassandra.db.compression;
import java.util.Map;
import com.google.common.base.Preconditions;
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.CompressionParams;
import static java.lang.String.format;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.DEFAULT_TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_VALUE;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.DEFAULT_TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_VALUE;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.DEFAULT_TRAINING_MIN_FREQUENCY;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.TRAINING_MIN_FREQUENCY_PARAMETER_NAME;
/**
* Configuration for dictionary training parameters.
*/
@ -29,6 +44,7 @@ public class CompressionDictionaryTrainingConfig
public final int maxTotalSampleSize;
public final int acceptableTotalSampleSize;
public final int chunkSize;
public final int minTrainingFrequency;
private CompressionDictionaryTrainingConfig(Builder builder)
{
@ -36,6 +52,7 @@ public class CompressionDictionaryTrainingConfig
this.maxTotalSampleSize = builder.maxTotalSampleSize;
this.acceptableTotalSampleSize = builder.maxTotalSampleSize / 10 * 8;
this.chunkSize = builder.chunkSize;
this.minTrainingFrequency = builder.minTrainingFrequency;
}
public static Builder builder()
@ -48,6 +65,7 @@ public class CompressionDictionaryTrainingConfig
private int maxDictionarySize = 65536; // 64KB default
private int maxTotalSampleSize = 10 * 1024 * 1024; // 10MB total
private int chunkSize = 64 * 1024; // 64KB default
private int minTrainingFrequency = 0; // in minutes
public Builder maxDictionarySize(int size)
{
@ -67,12 +85,121 @@ public class CompressionDictionaryTrainingConfig
return this;
}
public Builder minTrainingFrequency(int minTrainingFrequency)
{
this.minTrainingFrequency = minTrainingFrequency;
return this;
}
public CompressionDictionaryTrainingConfig build()
{
Preconditions.checkArgument(maxDictionarySize > 0, "maxDictionarySize must be positive");
Preconditions.checkArgument(maxTotalSampleSize > 0, "maxTotalSampleSize must be positive");
Preconditions.checkArgument(chunkSize > 0, "chunkSize must be positive");
Preconditions.checkArgument(minTrainingFrequency >= 0, "min training frequency must be non-negative");
return new CompressionDictionaryTrainingConfig(this);
}
}
public static int getMaxDictionarySize(Map<String, String> params)
{
return validateSizeBasedTrainingParameter(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME,
params.getOrDefault(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_VALUE));
}
public static int getMaxTotalSampleSize(Map<String, String> params)
{
return validateSizeBasedTrainingParameter(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME,
params.getOrDefault(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_VALUE));
}
public static int getMinTrainingFrequency(Map<String, String> params)
{
return validateDurationBasedTrainingParameter(TRAINING_MIN_FREQUENCY_PARAMETER_NAME,
params.getOrDefault(TRAINING_MIN_FREQUENCY_PARAMETER_NAME,
DEFAULT_TRAINING_MIN_FREQUENCY));
}
public static int getMaxDictionarySizeWithUserSuppliedParams(CompressionParams compressionParams, Map<String, String> parameters)
{
return internalTrainingParameterResolution(compressionParams,
parameters.get(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME),
TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_VALUE);
}
public static int getMaxTotalSampleSizeWithUserSuppliedParams(CompressionParams compressionParams, Map<String, String> parameters)
{
return internalTrainingParameterResolution(compressionParams,
parameters.get(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME),
TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_VALUE);
}
private static int internalTrainingParameterResolution(CompressionParams compressionParams,
String userSuppliedValue,
String parameterName,
String defaultParameterValue)
{
String resolvedValue = null;
try
{
if (userSuppliedValue == null)
resolvedValue = compressionParams.getOtherOptions().getOrDefault(parameterName, defaultParameterValue);
else
resolvedValue = userSuppliedValue;
return new DataStorageSpec.IntBytesBound(resolvedValue).toBytes();
}
catch (Throwable t)
{
throw new IllegalArgumentException(String.format("Invalid value for %s: %s", parameterName, resolvedValue));
}
}
/**
* Validates value of a parameter for training purposes. The value to validate should
* be accepted by {@link DataStorageSpec.IntKibibytesBound}. This method is used upon validation
* of input parameters in the implementations of dictionary compressor.
*
* @param parameterName name of a parameter to validate
* @param resolvedValue value to validate
* @return resolved value in bytes
*/
static int validateSizeBasedTrainingParameter(String parameterName, String resolvedValue)
{
try
{
return new DataStorageSpec.IntBytesBound(resolvedValue).toBytes();
}
catch (Throwable t)
{
throw new ConfigurationException(format("Unable to set value to parameter %s: %s. Reason: %s",
parameterName, resolvedValue, t.getMessage()));
}
}
/**
* Validates value of a parameter for training purposes. The value to validate should
* be accepted by {@link DurationSpec.IntMinutesBound}. This method is used upon validation of input parameters
* in the implementation of dictionary compressor.
*
* @param parameterName name of a parameter to validate
* @param resolvedValue value to validate
* @return resolved value in minutes
*/
static int validateDurationBasedTrainingParameter(String parameterName, String resolvedValue)
{
try
{
return new DurationSpec.IntMinutesBound(resolvedValue).toMinutes();
}
catch (Throwable t)
{
throw new ConfigurationException(format("Unable to set value to parameter %s: %s. Reason: %s",
parameterName, resolvedValue, t.getMessage()));
}
}
}

20
src/java/org/apache/cassandra/db/compression/ICompressionDictionaryScheduler.java
View File

@ -18,7 +18,10 @@
package org.apache.cassandra.db.compression;
import java.util.function.Consumer;
import org.apache.cassandra.db.ColumnFamilyStore.RefViewFragment;
import org.apache.cassandra.schema.CompressionParams;
/**
* Interface for managing scheduled tasks for compression dictionary operations.
@ -38,15 +41,24 @@ public interface ICompressionDictionaryScheduler extends AutoCloseable
/**
* Schedules SSTable-based training that samples from existing SSTables.
*
* @param trainer the trainer to use
* A caller of this method should ensure that SSTables referred in {@code refViewFragment} are closed
* eventually, either directly at the end of that method or by other means, when training is running
* asynchronously.
*
* A caller of this method might assume that {@code trainer} might be closed after this method finishes, either
* directly in this method or indirectly when training is running asynchronously.
*
* @param refViewFragment the view of SSTables to sample from
* @param compressionParams parameters for compression
* @param config the training configuration
* @param listener listener invoked when a dictionary is trained
* @param force force the dictionary training even if there are not enough samples
* @throws IllegalStateException if training is already in progress
*/
void scheduleSSTableBasedTraining(ICompressionDictionaryTrainer trainer,
RefViewFragment refViewFragment,
void scheduleSSTableBasedTraining(RefViewFragment refViewFragment,
CompressionParams compressionParams,
CompressionDictionaryTrainingConfig config,
Consumer<CompressionDictionary> listener,
boolean force);
/**
@ -55,4 +67,6 @@ public interface ICompressionDictionaryScheduler extends AutoCloseable
* @param enabled whether the scheduler should be enabled
*/
void setEnabled(boolean enabled);
TrainingState getLastTrainingState();
}

10
src/java/org/apache/cassandra/db/compression/ICompressionDictionaryTrainer.java
View File

@ -98,16 +98,6 @@ public interface ICompressionDictionaryTrainer extends AutoCloseable
*/
CompressionDictionary.Kind kind();
/**
* Determines if this trainer is compatible with the given compression parameters.
* This method allows the trainer to decide whether it can continue operating
* with new compression parameters or if a new trainer instance is needed.
*
* @param newParams the new compression parameters to check compatibility against
* @return true if this trainer is compatible with the new parameters, false otherwise
*/
boolean isCompatibleWith(CompressionParams newParams);
/**
* Sets the listener for dictionary training events.
*

87
src/java/org/apache/cassandra/db/compression/SSTableChunkSampler.java
View File

@ -35,6 +35,7 @@ import org.apache.cassandra.io.compress.ICompressor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.utils.ChecksumType;
import org.apache.cassandra.utils.memory.MemoryUtil;
/**
* Samples uncompressed chunks from existing SSTables for dictionary training.
@ -261,17 +262,23 @@ public class SSTableChunkSampler
long position = chunkIndex * info.chunkSize;
ByteBuffer chunk = readChunk(info, position);
// Check if adding this sample would exceed the max total sample size
if (totalSampleSize + chunk.remaining() > config.maxTotalSampleSize)
try
{
logger.debug("Next chunk would exceed max total sample size limit");
break;
}
// Check if adding this sample would exceed the max total sample size
if (totalSampleSize + chunk.remaining() > config.maxTotalSampleSize)
{
logger.debug("Next chunk would exceed max total sample size limit");
break;
}
trainer.addSample(chunk);
totalSampleSize += chunk.remaining();
sampleCount++;
trainer.addSample(chunk);
totalSampleSize += chunk.remaining();
sampleCount++;
}
finally
{
MemoryUtil.clean(chunk);
}
}
return new SamplingStats(sampleCount, totalSampleSize);
@ -373,38 +380,44 @@ public class SSTableChunkSampler
// Allocate buffer for compressed data + checksum
int compressedLength = chunk.length;
ByteBuffer compressed = ByteBuffer.allocateDirect(compressedLength + Integer.BYTES);
int read = channel.read(compressed, chunk.offset);
if (read != compressedLength + Integer.BYTES)
try
{
throw new IOException(String.format("Expected to read %d bytes but got %d",
compressedLength + Integer.BYTES, read));
int read = channel.read(compressed, chunk.offset);
if (read != compressedLength + Integer.BYTES)
{
throw new IOException(String.format("Expected to read %d bytes but got %d",
compressedLength + Integer.BYTES, read));
}
compressed.flip();
compressed.limit(compressedLength);
// Verify checksum
int expectedChecksum = (int) ChecksumType.CRC32.of(compressed);
compressed.limit(compressedLength + Integer.BYTES);
int actualChecksum = compressed.getInt(compressedLength);
if (expectedChecksum != actualChecksum)
{
throw new IOException(String.format("Checksum mismatch for chunk at position %d in SSTable %s (expected: %d, actual: %d)",
position, sstableInfo.sstable, expectedChecksum, actualChecksum));
}
// Reset for decompression
compressed.position(0).limit(compressedLength);
// Decompress the chunk
ICompressor compressor = metadata.compressor();
ByteBuffer uncompressed = ByteBuffer.allocateDirect(metadata.chunkLength());
compressor.uncompress(compressed, uncompressed);
uncompressed.flip();
return uncompressed;
}
compressed.flip();
compressed.limit(compressedLength);
// Verify checksum
int expectedChecksum = (int) ChecksumType.CRC32.of(compressed);
compressed.limit(compressedLength + Integer.BYTES);
int actualChecksum = compressed.getInt(compressedLength);
if (expectedChecksum != actualChecksum)
finally
{
throw new IOException(String.format("Checksum mismatch for chunk at position %d in SSTable %s (expected: %d, actual: %d)",
position, sstableInfo.sstable, expectedChecksum, actualChecksum));
MemoryUtil.clean(compressed);
}
// Reset for decompression
compressed.position(0).limit(compressedLength);
// Decompress the chunk
ICompressor compressor = metadata.compressor();
ByteBuffer uncompressed = ByteBuffer.allocateDirect(metadata.chunkLength());
compressor.uncompress(compressed, uncompressed);
uncompressed.flip();
return uncompressed;
}
/**

44
src/java/org/apache/cassandra/db/compression/ZstdDictionaryTrainer.java
View File

@ -32,10 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.db.compression.CompressionDictionary.DictId;
import org.apache.cassandra.db.compression.CompressionDictionary.Kind;
import org.apache.cassandra.io.compress.IDictionaryCompressor;
import org.apache.cassandra.io.compress.ZstdDictionaryCompressor;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import org.apache.cassandra.utils.concurrent.Future;
@ -298,8 +295,14 @@ public class ZstdDictionaryTrainer implements ICompressionDictionaryTrainer
}
catch (Exception e)
{
logger.warn("Failed to create ZstdDictTrainer for {}.{}", keyspaceName, tableName, e);
failureMessage = "Failed to create ZstdDictTrainer: " + e.getMessage();
String message = String.format("Failed to create %s for %s.%s, reason: %s",
ZstdDictTrainer.class.getSimpleName(),
keyspaceName,
tableName,
e.getMessage());
logger.warn(message);
failureMessage = message;
currentTrainingStatus = TrainingStatus.FAILED;
}
return false;
@ -318,7 +321,15 @@ public class ZstdDictionaryTrainer implements ICompressionDictionaryTrainer
{
totalSampleSize.set(0);
sampleCount.set(0);
zstdTrainer = new ZstdDictTrainer(trainingConfig.maxTotalSampleSize, trainingConfig.maxDictionarySize, compressionLevel);
try
{
zstdTrainer = new ZstdDictTrainer(trainingConfig.maxTotalSampleSize, trainingConfig.maxDictionarySize, compressionLevel);
}
catch (Throwable t)
{
throw new IllegalStateException(t);
}
config = trainingConfig;
}
}
@ -356,27 +367,6 @@ public class ZstdDictionaryTrainer implements ICompressionDictionaryTrainer
}
}
@Override
public boolean isCompatibleWith(CompressionParams newParams)
{
if (!newParams.isDictionaryCompressionEnabled())
{
return false;
}
IDictionaryCompressor newCompressor = (IDictionaryCompressor) newParams.getSstableCompressor();
// Check if the compressor type is compatible with this trainer
if (newCompressor.acceptableDictionaryKind() != Kind.ZSTD)
{
return false;
}
ZstdDictionaryCompressor zstdDictionaryCompressor = (ZstdDictionaryCompressor) newCompressor;
// For Zstd compressors, check if compression level matches
return this.compressionLevel == zstdDictionaryCompressor.compressionLevel();
}
@Override
public void close()
{

47
src/java/org/apache/cassandra/io/compress/IDictionaryCompressor.java
View File

@ -18,12 +18,7 @@
package org.apache.cassandra.io.compress;
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.compression.CompressionDictionary;
import org.apache.cassandra.exceptions.ConfigurationException;
import static java.lang.String.format;
/**
* Interface for compressors that support dictionary-based compression.
@ -47,48 +42,6 @@ public interface IDictionaryCompressor<T extends CompressionDictionary>
// that once we train a dictionary for given table, then we can train again after at least 1 hour.
String DEFAULT_TRAINING_MIN_FREQUENCY = "0m";
/**
* Validates value of a parameter for training purposes. The value to validate should
* be accepted by {@link DataStorageSpec.IntKibibytesBound}. This method is used upon validation
* of input parameters in the implementations of dictionary compressor.
*
* @param parameterName name of a parameter to validate
* @param resolvedValue value to validate
*/
static void validateSizeBasedTrainingParameter(String parameterName, String resolvedValue)
{
try
{
new DataStorageSpec.IntBytesBound(resolvedValue).toBytes();
}
catch (Throwable t)
{
throw new ConfigurationException(format("Unable to set value to parameter %s: %s. Reason: %s",
parameterName, resolvedValue, t.getMessage()));
}
}
/**
* Validates value of a parameter for training purposes. The value to validate should
* be accepted by {@link DurationSpec.IntMinutesBound}. This method is used upon validation of input parameters
* in the implementation of dictionary compressor.
*
* @param parameterName name of a parameter to validate
* @param resolvedValue value to validate
*/
static void validateDurationBasedTrainingParameter(String parameterName, String resolvedValue)
{
try
{
new DurationSpec.IntMinutesBound(resolvedValue).toMinutes();
}
catch (Throwable t)
{
throw new ConfigurationException(format("Unable to set value to parameter %s: %s. Reason: %s",
parameterName, resolvedValue, t.getMessage()));
}
}
/**
* Returns a compressor instance configured with the specified compression dictionary.
* <br>

17
src/java/org/apache/cassandra/io/compress/ZstdDictionaryCompressor.java
View File

@ -36,12 +36,10 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.compression.CompressionDictionary.Kind;
import org.apache.cassandra.db.compression.CompressionDictionaryTrainingConfig;
import org.apache.cassandra.db.compression.ZstdCompressionDictionary;
import org.apache.cassandra.utils.concurrent.Ref;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.validateDurationBasedTrainingParameter;
import static org.apache.cassandra.io.compress.IDictionaryCompressor.validateSizeBasedTrainingParameter;
public class ZstdDictionaryCompressor extends ZstdCompressorBase implements ICompressor, IDictionaryCompressor<ZstdCompressionDictionary>
{
private static final ConcurrentHashMap<Integer, ZstdDictionaryCompressor> instancesPerLevel = new ConcurrentHashMap<>();
@ -78,15 +76,10 @@ public class ZstdDictionaryCompressor extends ZstdCompressorBase implements ICom
{
int level = getOrDefaultCompressionLevel(options);
validateCompressionLevel(level);
validateSizeBasedTrainingParameter(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME,
options.getOrDefault(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_VALUE));
validateSizeBasedTrainingParameter(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME,
options.getOrDefault(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME,
DEFAULT_TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_VALUE));
validateDurationBasedTrainingParameter(TRAINING_MIN_FREQUENCY_PARAMETER_NAME,
options.getOrDefault(TRAINING_MIN_FREQUENCY_PARAMETER_NAME,
DEFAULT_TRAINING_MIN_FREQUENCY));
// pass it through to validate
CompressionDictionaryTrainingConfig.getMaxDictionarySize(options);
CompressionDictionaryTrainingConfig.getMaxTotalSampleSize(options);
CompressionDictionaryTrainingConfig.getMinTrainingFrequency(options);
return getOrCreate(level, null);
}

14
src/java/org/apache/cassandra/schema/CompressionParams.java
View File

@ -33,6 +33,7 @@ import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.compression.CompressionDictionary;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.compress.DeflateCompressor;
@ -258,6 +259,19 @@ public final class CompressionParams
return isEnabled() && sstableCompressor instanceof IDictionaryCompressor;
}
/**
* @return kind of compression dictionary the compressor accepts, or null if none
*/
public CompressionDictionary.Kind getCompressionDictionaryKind()
{
if (isDictionaryCompressionEnabled())
{
return ((IDictionaryCompressor<?>) sstableCompressor).acceptableDictionaryKind();
}
return null;
}
/**
* Returns the SSTable compressor.
* @return the SSTable compressor or {@code null} if compression is disabled.

6
src/java/org/apache/cassandra/tools/nodetool/CompressionDictionaryCommandGroup.java
View File

@ -31,9 +31,9 @@ import javax.management.openmbean.TabularData;
import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.db.compression.CompressionDictionaryDetailsTabularData;
import org.apache.cassandra.db.compression.CompressionDictionaryDetailsTabularData.CompressionDictionaryDataObject;
import org.apache.cassandra.db.compression.CompressionDictionaryTrainingConfig;
import org.apache.cassandra.db.compression.ICompressionDictionaryTrainer.TrainingStatus;
import org.apache.cassandra.db.compression.TrainingState;
import org.apache.cassandra.io.util.File;
@ -182,7 +182,7 @@ public class CompressionDictionaryCommandGroup
{
try
{
new DataStorageSpec.IntBytesBound(trainingMaxDictionarySize).toBytes();
CompressionDictionaryTrainingConfig.getMaxDictionarySize(Map.of(TRAINING_MAX_DICTIONARY_SIZE_PARAMETER_NAME, trainingMaxDictionarySize));
}
catch (Throwable t)
{
@ -195,7 +195,7 @@ public class CompressionDictionaryCommandGroup
{
try
{
new DataStorageSpec.IntBytesBound(trainingMaxTotalSampleSize).toBytes();
CompressionDictionaryTrainingConfig.getMaxTotalSampleSize(Map.of(TRAINING_MAX_TOTAL_SAMPLE_SIZE_PARAMETER_NAME, trainingMaxTotalSampleSize));
}
catch (Throwable t)
{

28
test/unit/org/apache/cassandra/db/compression/CompressionDictionaryIntegrationTest.java
View File

@ -18,7 +18,6 @@
package org.apache.cassandra.db.compression;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -113,28 +112,7 @@ public class CompressionDictionaryIntegrationTest extends CQLTester
}
@Test
public void testCompressionParameterChanges()
{
String table = createTable(getTableCql());
ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(table);
CompressionDictionaryManager manager = cfs.compressionDictionaryManager();
ICompressionDictionaryTrainer trainer = manager.trainer();
assertThat(trainer).isNotNull();
assertThat(trainer.kind()).isEqualTo(Kind.ZSTD);
// Change compression level - should create new trainer
CompressionParams newParams = CompressionParams.zstd(CompressionParams.DEFAULT_CHUNK_LENGTH, true,
Collections.singletonMap("compression_level", "5"));
manager.maybeReloadFromSchema(newParams);
ICompressionDictionaryTrainer newTrainer = manager.trainer();
assertThat(newTrainer.kind()).isEqualTo(Kind.ZSTD);
assertThat(newTrainer)
.as("Should create a different trainer instance when compression level is changed")
.isNotSameAs(trainer);
}
@Test
public void testResourceCleanupOnClose() throws Exception
public void testResourceCleanupOnClose()
{
createTable(getTableCql());
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
@ -154,7 +132,9 @@ public class CompressionDictionaryIntegrationTest extends CQLTester
manager.close();
assertThat(manager.trainer()).isNull();
assertThat(TrainingState.fromCompositeData(manager.getTrainingState()).status)
.isEqualTo(TrainingState.notStarted().status);
// Dictionary's reference count should be 0 after closing manager
// Dictionary is closed in a separate thread. Wait a bit for the reference count to be updated.
spinUntilTrue(() -> testDict.selfRef().globalCount() == 0, 1, TimeUnit.SECONDS);

46
test/unit/org/apache/cassandra/db/compression/CompressionDictionaryManagerTest.java
View File

@ -116,7 +116,7 @@ public class CompressionDictionaryManagerTest
TrainingState trainingState = TrainingState.fromCompositeData(managerWithDict.getTrainingState());
assertThat(trainingState.getStatus())
.as("Training status should be valid")
.isEqualTo(TrainingStatus.SAMPLING);
.isEqualTo(TrainingStatus.NOT_STARTED);
}
@Test
@ -148,44 +148,28 @@ public class CompressionDictionaryManagerTest
managerWithoutDict.maybeReloadFromSchema(dictParams);
// Should now have a trainer
assertThat(managerWithoutDict.trainer())
.as("Should have a trainer after enabling dictionary compression")
.isNotNull();
// Should be now enabled
assertThat(managerWithoutDict.isEnabled())
.as("manager should have enabled dictionary compression")
.isTrue();
}
@Test
public void testMaybeReloadFromSchemaDisableDictionaryCompression()
{
// Verify we have a trainer initially
assertThat(managerWithDict.trainer()).isNotNull();
assertThat(managerWithDict.isEnabled())
.as("manager with a dictionary support should be enabled")
.isTrue();
// Disable dictionary compression
CompressionParams nonDictParams = CompressionParams.lz4();
managerWithDict.maybeReloadFromSchema(nonDictParams);
// Should disable training
assertThat(managerWithDict.trainer())
.as("Should not have trainer when dictionary compression is disabled")
.isNull();
}
@Test
public void testTrainerCompatibilityCheck()
{
ICompressionDictionaryTrainer initialTrainer = managerWithDict.trainer();
assertThat(initialTrainer).isNotNull();
// Change compression level - should create new trainer
CompressionParams differentLevelParams = CompressionParams.zstd(CompressionParams.DEFAULT_CHUNK_LENGTH, true,
Map.of("compression_level", "5"));
managerWithDict.maybeReloadFromSchema(differentLevelParams);
ICompressionDictionaryTrainer newTrainer = managerWithDict.trainer();
// Should have a different trainer instance
assertThat(newTrainer)
.as("Should create new trainer when compression level changes")
.isNotSameAs(initialTrainer);
assertThat(managerWithDict.isEnabled())
.as("manager without a dictionary support should be disabled")
.isFalse();
}
@Test
@ -211,7 +195,7 @@ public class CompressionDictionaryManagerTest
// Start with non-dictionary table
TrainingState initialTrainingState = TrainingState.fromCompositeData(managerWithoutDict.getTrainingState());
assertThat(initialTrainingState.getStatus()).isEqualTo(TrainingStatus.NOT_STARTED);
assertThat(managerWithoutDict.trainer()).isNull();
assertThat(managerWithoutDict.isEnabled()).isFalse();
// Enable dictionary compression
CompressionParams dictParams = CompressionParams.zstd(CompressionParams.DEFAULT_CHUNK_LENGTH, true,
@ -219,7 +203,7 @@ public class CompressionDictionaryManagerTest
managerWithoutDict.maybeReloadFromSchema(dictParams);
// Should now support training
assertThat(managerWithoutDict.trainer()).isNotNull();
assertThat(managerWithoutDict.isEnabled()).isTrue();
// Change compression level
CompressionParams newDictParams = CompressionParams.zstd(CompressionParams.DEFAULT_CHUNK_LENGTH, true,
@ -227,13 +211,13 @@ public class CompressionDictionaryManagerTest
managerWithoutDict.maybeReloadFromSchema(newDictParams);
// Should still support training with new parameters
assertThat(managerWithoutDict.trainer()).isNotNull();
assertThat(managerWithoutDict.isEnabled()).isTrue();
// Disable dictionary compression
CompressionParams nonDictParams = CompressionParams.lz4();
managerWithoutDict.maybeReloadFromSchema(nonDictParams);
// Should disable training
assertThat(managerWithoutDict.trainer()).isNull();
assertThat(managerWithoutDict.isEnabled()).isFalse();
}
}

9
test/unit/org/apache/cassandra/db/compression/CompressionDictionarySchedulerTest.java
View File

@ -68,8 +68,8 @@ public class CompressionDictionarySchedulerTest extends CQLTester
CompressionDictionaryTrainingConfig config = createSampleAllTrainingConfig(cfs);
// Should not throw, but task will complete quickly with no SSTables
scheduler.scheduleSSTableBasedTraining(manager.trainer(), refViewFragment, config, true);
spinUntilTrue(() -> !scheduler.isManualTrainingRunning());
scheduler.scheduleSSTableBasedTraining(refViewFragment, cfs.metadata.get().params.compression, config, manager::handleNewDictionary, true);
spinUntilTrue(() -> !scheduler.isTrainingRunning());
assertThat(manager.getCurrent()).isNull();
}
}
@ -91,13 +91,12 @@ public class CompressionDictionarySchedulerTest extends CQLTester
assertThat(refViewFragment.sstables).isNotEmpty();
CompressionDictionaryTrainingConfig config = createSampleAllTrainingConfig(cfs);
manager.trainer().start(config);
assertThat(manager.getCurrent()).as("There should be no dictionary at this step").isNull();
scheduler.scheduleSSTableBasedTraining(manager.trainer(), refViewFragment, config, true);
scheduler.scheduleSSTableBasedTraining(refViewFragment, cfs.metadata.get().params.compression, config, manager::handleNewDictionary, true);
// Task should be scheduled
assertThat(scheduler.isManualTrainingRunning()).isTrue();
assertThat(scheduler.isTrainingRunning()).isTrue();
// A dictionary should be trained
spinUntilTrue(() -> manager.getCurrent() != null);
}

33
test/unit/org/apache/cassandra/db/compression/ZstdDictionaryTrainerTest.java
View File

@ -19,7 +19,6 @@
package org.apache.cassandra.db.compression;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@ -32,7 +31,6 @@ import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compression.ICompressionDictionaryTrainer.TrainingStatus;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.Future;
@ -442,37 +440,6 @@ public class ZstdDictionaryTrainerTest
assertThat(id3).isNotEqualTo(id1).isNotEqualTo(id2);
}
@Test
public void testIsCompatibleWith()
{
CompressionParams compatibleParams = CompressionParams.zstd(CompressionParams.DEFAULT_CHUNK_LENGTH, true,
Map.of("compression_level", "3"));
assertThat(trainer.isCompatibleWith(compatibleParams))
.as("Should be compatible with same compression level")
.isTrue();
CompressionParams incompatibleParams = CompressionParams.lz4();
assertThat(trainer.isCompatibleWith(incompatibleParams))
.as("Should not be compatible with different compressor")
.isFalse();
CompressionParams differentLevelParams = CompressionParams.zstd(CompressionParams.DEFAULT_CHUNK_LENGTH, true,
Map.of("compression_level", "4"));
assertThat(trainer.isCompatibleWith(differentLevelParams))
.as("Should not be compatible with different compression level")
.isFalse();
CompressionParams disabledParams = CompressionParams.noCompression();
assertThat(trainer.isCompatibleWith(disabledParams))
.as("Should not be compatible with disabled compression")
.isFalse();
}
@Test
public void testTrainDictionaryNotInitialized()
{