natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix compression test runs 

Ensure the specified commitlog_compression_xxx.yaml file is appended to the used cassandra.compressed.yaml, according to the selected algorithm as defined by `test.compression.algo` the property. By default we use the default LZ4. An unknown compression algo with fail.

 patch by Jacek Lewandowski; reviewed by Mick Semb Wever for CASSANDRA-16575
This commit is contained in:
jacek-lewandowski 2021-04-23 09:15:22 +02:00 committed by Mick Semb Wever
parent 5578627d74
commit 10a1d65eb0
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
2 changed files with 29 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
build.xml
View File

@ -61,7 +61,7 @@
<property name="test.memory.src" value="${test.dir}/memory"/>
<property name="test.microbench.src" value="${test.dir}/microbench"/>
<property name="test.distributed.src" value="${test.dir}/distributed"/>
<property name="test.compression_algo" value="LZ4"/>
<property name="test.compression.algo" value="LZ4"/>
<property name="test.driver.connection_timeout_ms" value="5000"/>
<property name="test.driver.read_timeout_ms" value="12000"/>
<property name="dist.dir" value="${build.dir}/dist"/>
@ -1430,13 +1430,14 @@
<property name="compressed_yaml" value="${build.test.dir}/cassandra.compressed.yaml"/>
<concat destfile="${compressed_yaml}">
<fileset file="${test.conf}/cassandra.yaml"/>
<fileset file="${test.conf}/commitlog_compression.yaml"/>
<fileset file="${test.conf}/commitlog_compression_${test.compression.algo}.yaml"/>
</concat>
<testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}"
exclude="**/*.java" timeout="${test.timeout}" testtag="compression">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
<jvmarg value="-Dcassandra.test.compression=true"/>
<jvmarg value="-Dcassandra.test.compression.algo=${test.compression.algo}"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.config=file:///${compressed_yaml}"/>

34
test/unit/org/apache/cassandra/SchemaLoader.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.cassandra;
import java.io.File;
import java.io.IOException;
import java.util.*;
@ -28,12 +27,10 @@ import org.apache.cassandra.auth.IAuthorizer;
import org.apache.cassandra.auth.INetworkAuthorizer;
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
@ -41,7 +38,6 @@ import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.index.StubIndex;
import org.apache.cassandra.index.sasi.SASIIndex;
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -248,7 +244,7 @@ public class SchemaLoader
MigrationManager.announceNewKeyspace(ksm, false);
if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
useCompression(schema);
useCompression(schema, compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH));
}
public static void createKeyspace(String name, KeyspaceParams params)
@ -329,11 +325,11 @@ public class SchemaLoader
return builder.build();
}
private static void useCompression(List<KeyspaceMetadata> schema)
private static void useCompression(List<KeyspaceMetadata> schema, CompressionParams compressionParams)
{
for (KeyspaceMetadata ksm : schema)
for (TableMetadata cfm : ksm.tablesAndViews())
MigrationManager.announceTableUpdate(cfm.unbuild().compression(CompressionParams.snappy()).build(), true);
MigrationManager.announceTableUpdate(cfm.unbuild().compression(compressionParams.copy()).build(), true);
}
public static TableMetadata.Builder counterCFMD(String ksName, String cfName)
@ -721,7 +717,7 @@ public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfN
public static CompressionParams getCompressionParameters(Integer chunkSize)
{
if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
return chunkSize != null ? CompressionParams.snappy(chunkSize) : CompressionParams.snappy();
return chunkSize != null ? compressionParams(chunkSize) : compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH);
return CompressionParams.noCompression();
}
@ -751,4 +747,26 @@ public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfN
{
ServerTestUtils.cleanupSavedCaches();
}
private static CompressionParams compressionParams(int chunkLength)
{
String algo = System.getProperty("cassandra.test.compression.algo", "lz4").toLowerCase();
switch (algo)
{
case "deflate":
return CompressionParams.deflate(chunkLength);
case "lz4":
return CompressionParams.lz4(chunkLength);
case "snappy":
return CompressionParams.snappy(chunkLength);
case "noop":
return CompressionParams.noop();
case "zstd":
return CompressionParams.zstd(chunkLength);
case "none":
return CompressionParams.noCompression();
default:
throw new IllegalArgumentException("Invalid compression algorithm has been provided in cassandra.test.compression system property: " + algo);
}
}
}