natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-5.0' into trunk

This commit is contained in:
Stefan Miklosovic 2023-09-29 15:19:20 +02:00
parent fbf6508940 b9f614f7e5
commit fdbb975188
No known key found for this signature in database
GPG Key ID: 32F35CB2F546D93E
13 changed files with 39 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -3,6 +3,7 @@
* Add the ability to disable bulk loading of SSTables (CASSANDRA-18781)
* Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787)
Merged from 5.0:
* Remove deprecated properties in CompressionParams (CASSANDRA-18742)
* Add support for repair coordinator to retry messages that timeout (CASSANDRA-18816)
* Upgrade slf4j-api to 1.7.36 (CASSANDRA-18882)
* Make the output of ON/OFF commands in cqlsh consistent (CASSANDRA-18547)

6
pylib/cqlshlib/cql3handling.py
View File

@ -64,7 +64,7 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
('class', 'max_threshold', 'tombstone_compaction_interval', 'tombstone_threshold', 'enabled',
'unchecked_tombstone_compaction', 'only_purge_repaired_tombstones', 'provide_overlapping_tombstones')),
('compression', 'compression_parameters',
('sstable_compression', 'chunk_length_kb', 'crc_check_chance')),
('class', 'chunk_length_in_kb', 'enabled', 'min_compress_ratio', 'max_compressed_length')),
('caching', None,
('rows_per_partition', 'keys')),
)
@ -498,7 +498,7 @@ def cf_prop_val_completer(ctxt, cass):
exist_opts = ctxt.get_binding('propname')
this_opt = exist_opts[-1]
if this_opt == 'compression':
return ["{'sstable_compression': '"]
return ["{'class': '"]
if this_opt == 'compaction':
return ["{'class': '"]
if this_opt == 'caching':
@ -563,7 +563,7 @@ def cf_prop_val_mapval_completer(ctxt, cass):
return [Hint('<NONE|ROW|CELL>')]
return [Hint('<option_value>')]
elif opt == 'compression':
if key == 'sstable_compression':
if key == 'class':
return list(map(escape_value, CqlRuleSet.available_compression_classes))
return [Hint('<option_value>')]
elif opt == 'caching':

2
pylib/cqlshlib/test/test_keyspace_init.cql
View File

@ -34,7 +34,7 @@ CREATE TABLE has_all_types (
uuidcol uuid,
varcharcol varchar,
varintcol varint
) WITH compression = {'sstable_compression':'LZ4Compressor'};
) WITH compression = {'class':'LZ4Compressor'};
INSERT INTO has_all_types (num, intcol, asciicol, bigintcol, blobcol, booleancol,
decimalcol, doublecol, durationcol, floatcol, smallintcol, textcol,

74
src/java/org/apache/cassandra/schema/CompressionParams.java
View File

@ -50,8 +50,6 @@ public final class CompressionParams
{
private static final Logger logger = LoggerFactory.getLogger(CompressionParams.class);
private static volatile boolean hasLoggedSsTableCompressionWarning;
private static volatile boolean hasLoggedChunkLengthWarning;
private static volatile boolean hasLoggedCrcCheckChanceWarning;
public static final int DEFAULT_CHUNK_LENGTH = 1024 * 16;
@ -83,8 +81,6 @@ public final class CompressionParams
private static final String CRC_CHECK_CHANCE_WARNING = "The option crc_check_chance was deprecated as a compression option. " +
"You should specify it as a top-level table option instead";
@Deprecated public static final String SSTABLE_COMPRESSION = "sstable_compression";
@Deprecated public static final String CHUNK_LENGTH_KB = "chunk_length_kb";
@Deprecated public static final String CRC_CHECK_CHANCE = "crc_check_chance";
private final ICompressor sstableCompressor;
@ -102,20 +98,13 @@ public final class CompressionParams
String sstableCompressionClass;
if (!opts.isEmpty() && isEnabled(opts) && !containsSstableCompressionClass(opts))
if (!opts.isEmpty() && isEnabled(opts) && !options.containsKey(CLASS))
throw new ConfigurationException(format("Missing sub-option '%s' for the 'compression' option.", CLASS));
if (!removeEnabled(options))
{
sstableCompressionClass = null;
if (!options.isEmpty())
throw new ConfigurationException(format("If the '%s' option is set to false no other options must be specified", ENABLED));
}
if (!removeEnabled(options) && !options.isEmpty())
throw new ConfigurationException(format("If the '%s' option is set to false no other options must be specified", ENABLED));
else
{
sstableCompressionClass = removeSstableCompressionClass(options);
}
sstableCompressionClass = removeSSTableCompressionClass(options);
int chunkLength = removeChunkLength(options);
double minCompressRatio = removeMinCompressRatio(options);
@ -133,7 +122,7 @@ public final class CompressionParams
public static CompressionParams noCompression()
{
return new CompressionParams((ICompressor) null, DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, 0.0, Collections.emptyMap());
return new CompressionParams(null, DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, 0.0, Collections.emptyMap());
}
// The shorthand methods below are only used for tests. They are a little inconsistent in their choice of
@ -403,29 +392,9 @@ public final class CompressionParams
{
if (options.containsKey(CHUNK_LENGTH_IN_KB))
{
if (options.containsKey(CHUNK_LENGTH_KB))
{
throw new ConfigurationException(format("The '%s' option must not be used if the chunk length is already specified by the '%s' option",
CHUNK_LENGTH_KB,
CHUNK_LENGTH_IN_KB));
}
return parseChunkLength(options.remove(CHUNK_LENGTH_IN_KB));
}
if (options.containsKey(CHUNK_LENGTH_KB))
{
if (!hasLoggedChunkLengthWarning)
{
hasLoggedChunkLengthWarning = true;
logger.warn("The {} option has been deprecated. You should use {} instead",
CHUNK_LENGTH_KB,
CHUNK_LENGTH_IN_KB);
}
return parseChunkLength(options.remove(CHUNK_LENGTH_KB));
}
return DEFAULT_CHUNK_LENGTH;
}
@ -445,50 +414,25 @@ public final class CompressionParams
return DEFAULT_MIN_COMPRESS_RATIO;
}
/**
* Returns {@code true} if the specified options contains the name of the compression class to be used,
* {@code false} otherwise.
*
* @param options the options
* @return {@code true} if the specified options contains the name of the compression class to be used,
* {@code false} otherwise.
*/
public static boolean containsSstableCompressionClass(Map<String, String> options)
{
return options.containsKey(CLASS) || options.containsKey(SSTABLE_COMPRESSION);
}
/**
* Removes the option specifying the name of the compression class
*
* @param options the options
* @return the name of the compression class
*/
private static String removeSstableCompressionClass(Map<String, String> options)
private static String removeSSTableCompressionClass(Map<String, String> options)
{
if (options.containsKey(CLASS))
{
if (options.containsKey(SSTABLE_COMPRESSION))
throw new ConfigurationException(format("The '%s' option must not be used if the compression algorithm is already specified by the '%s' option",
SSTABLE_COMPRESSION,
CLASS));
String clazz = options.remove(CLASS);
if (clazz.isEmpty())
if (clazz == null || clazz.isEmpty())
throw new ConfigurationException(format("The '%s' option must not be empty. To disable compression use 'enabled' : false", CLASS));
return clazz;
}
if (options.containsKey(SSTABLE_COMPRESSION) && !hasLoggedSsTableCompressionWarning)
{
hasLoggedSsTableCompressionWarning = true;
logger.warn("The {} option has been deprecated. You should use {} instead",
SSTABLE_COMPRESSION,
CLASS);
}
return options.remove(SSTABLE_COMPRESSION);
return null;
}
/**

2
test/distributed/org/apache/cassandra/distributed/util/PyDtest.java
View File

@ -160,7 +160,7 @@ public class PyDtest
query += String.format(" AND CLUSTERING ORDER BY (%s)", clustering);
if (compression != null)
query += String.format(" AND compression = { \'sstable_compression\': \'%sCompressor\' }", compression);
query += String.format(" AND compression = { \'class\': \'%sCompressor\' }", compression);
else
query += " AND compression = {}";

2
test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java
View File

@ -203,7 +203,7 @@ public class CachingBenchTest extends CQLTester
DatabaseDescriptor.setFileCacheEnabled(cacheEnabled);
DatabaseDescriptor.setDiskAccessMode(mode);
alterTable("ALTER TABLE %s WITH compaction = { 'class' : '" + compactionClass + "' };");
alterTable("ALTER TABLE %s WITH compression = { 'sstable_compression' : '" + compressorClass + "' };");
alterTable("ALTER TABLE %s WITH compression = { 'class' : '" + compressorClass + "' };");
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
cfs.disableAutoCompaction();

10
test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java
View File

@ -58,9 +58,9 @@ public class CrcCheckChanceTest extends CQLTester
{
//Start with crc_check_chance of 99%
if (newFormat)
createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'sstable_compression': 'LZ4Compressor'} AND crc_check_chance = 0.99;");
createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'class': 'LZ4Compressor'} AND crc_check_chance = 0.99;");
else
createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance' : 0.99}");
createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'class': 'LZ4Compressor', 'crc_check_chance' : 0.99}");
execute("CREATE INDEX foo ON %s(v)");
@ -82,7 +82,7 @@ public class CrcCheckChanceTest extends CQLTester
if (newFormat)
alterTable("ALTER TABLE %s WITH crc_check_chance = 0.99");
else
alterTable("ALTER TABLE %s WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance': 0.99}");
alterTable("ALTER TABLE %s WITH compression = {'class': 'LZ4Compressor', 'crc_check_chance': 0.99}");
assertRows(execute("SELECT * FROM %s WHERE p=?", "p1"),
row("p1", "k1", "sv1", "v1"),
@ -134,7 +134,7 @@ public class CrcCheckChanceTest extends CQLTester
if (newFormat)
alterTable("ALTER TABLE %s WITH crc_check_chance = 0.5");
else
alterTable("ALTER TABLE %s WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance': 0.5}");
alterTable("ALTER TABLE %s WITH compression = {'class': 'LZ4Compressor', 'crc_check_chance': 0.5}");
//We should be able to get the new value by accessing directly the schema metadata
Assert.assertEquals(0.5, cfs.metadata().params.crcCheckChance, 0.0);
@ -173,7 +173,7 @@ public class CrcCheckChanceTest extends CQLTester
CompactionManager.instance.disableAutoCompaction();
//Start with crc_check_chance of 99%
createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'sstable_compression': 'LZ4Compressor', 'crc_check_chance' : 0.99}");
createTable("CREATE TABLE %s (p text, c text, v text, s text static, PRIMARY KEY (p, c)) WITH compression = {'class': 'LZ4Compressor', 'crc_check_chance' : 0.99}");
ColumnFamilyStore cfs = Keyspace.open(CQLTester.KEYSPACE).getColumnFamilyStore(currentTable());

9
test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
View File

@ -710,18 +710,11 @@ public class AlterTest extends CQLTester
"The 'class' option must not be empty. To disable compression use 'enabled' : false",
"ALTER TABLE %s WITH compression = { 'class' : ''};");
assertAlterTableThrowsException(ConfigurationException.class,
"If the 'enabled' option is set to false no other options must be specified",
"ALTER TABLE %s WITH compression = { 'enabled' : 'false', 'class' : 'SnappyCompressor'};");
assertAlterTableThrowsException(ConfigurationException.class,
"The 'sstable_compression' option must not be used if the compression algorithm is already specified by the 'class' option",
"ALTER TABLE %s WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'class' : 'SnappyCompressor'};");
assertAlterTableThrowsException(ConfigurationException.class,
"The 'chunk_length_kb' option must not be used if the chunk length is already specified by the 'chunk_length_in_kb' option",
"ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };");
assertAlterTableThrowsException(ConfigurationException.class,
"Invalid negative min_compress_ratio",
"ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : -1 };");

22
test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
View File

@ -681,25 +681,17 @@ public class CreateTest extends CQLTester
assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'chunk_length_kb' : 32 };");
assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 2 };");
+ " WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : 2 };");
assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "2.0"));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 1 };");
+ " WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : 1 };");
assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.0"));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 0 };");
+ " WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : 0 };");
assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor"));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : '', 'chunk_length_kb' : 32 };");
assertSchemaOption("compression", map("enabled", "false"));
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'enabled' : 'false'};");
assertSchemaOption("compression", map("enabled", "false"));
@ -720,14 +712,6 @@ public class CreateTest extends CQLTester
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'enabled' : 'false', 'chunk_length_in_kb' : 32};");
assertThrowsConfigurationException("The 'sstable_compression' option must not be used if the compression algorithm is already specified by the 'class' option",
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'class' : 'SnappyCompressor'};");
assertThrowsConfigurationException("The 'chunk_length_kb' option must not be used if the chunk length is already specified by the 'chunk_length_in_kb' option",
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };");
assertThrowsConfigurationException("chunk_length_in_kb must be a power of 2",
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
+ " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 31 };");

8
test/unit/org/apache/cassandra/db/guardrails/GuardrailAllowUncompressedTablesTest.java
View File

@ -39,7 +39,7 @@ public class GuardrailAllowUncompressedTablesTest extends GuardrailTester
{
setGuardrail(true);
String table = createTableName();
schemaChange(String.format("CREATE TABLE %s.%s (k int primary key, v int) WITH compression={'sstable_compression':''}", KEYSPACE, table));
schemaChange(String.format("CREATE TABLE %s.%s (k int primary key, v int) WITH compression={'enabled':false}", KEYSPACE, table));
TableMetadata tmd = Schema.instance.getTableMetadata(KEYSPACE, table);
Assert.assertFalse(tmd.params.compression.isEnabled());
}
@ -52,7 +52,7 @@ public class GuardrailAllowUncompressedTablesTest extends GuardrailTester
{
setGuardrail(false);
String table = createTableName();
assertFails(String.format("CREATE TABLE %s.%s (k int primary key, v int) WITH compression={'sstable_compression':''}", KEYSPACE, table), "Uncompressed table is not allowed");
assertFails(String.format("CREATE TABLE %s.%s (k int primary key, v int) WITH compression={'enabled': false}", KEYSPACE, table), "Uncompressed table is not allowed");
}
@Test
@ -61,7 +61,7 @@ public class GuardrailAllowUncompressedTablesTest extends GuardrailTester
setGuardrail(true);
String table = createTableName();
schemaChange(String.format("CREATE TABLE %s.%s (k int primary key, v int)", KEYSPACE, table));
schemaChange(String.format("ALTER TABLE %s.%s WITH compression = {'sstable_compression': ''}", KEYSPACE, table));
schemaChange(String.format("ALTER TABLE %s.%s WITH compression = {'enabled': false}", KEYSPACE, table));
TableMetadata tmd = Schema.instance.getTableMetadata(KEYSPACE, table);
Assert.assertFalse(tmd.params.compression.isEnabled());
}
@ -72,6 +72,6 @@ public class GuardrailAllowUncompressedTablesTest extends GuardrailTester
setGuardrail(false);
String table = createTableName();
schemaChange(String.format("CREATE TABLE %s.%s (k int primary key, v int)", KEYSPACE, table));
assertFails(String.format("ALTER TABLE %s.%s WITH compression = {'sstable_compression': ''}", KEYSPACE, table), "Uncompressed table is not allowed");
assertFails(String.format("ALTER TABLE %s.%s WITH compression = {'enabled': false}", KEYSPACE, table), "Uncompressed table is not allowed");
}
}

14
test/unit/org/apache/cassandra/io/compress/CQLCompressionTest.java
View File

@ -106,7 +106,7 @@ public class CQLCompressionTest extends CQLTester
@Test
public void lz4FlushTest() throws Throwable
{
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'LZ4Compressor'};");
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'LZ4Compressor'};");
ColumnFamilyStore store = flushTwice();
// Should flush as LZ4 "fast"
@ -131,7 +131,7 @@ public class CQLCompressionTest extends CQLTester
public void lz4hcFlushTest() throws Throwable
{
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = " +
"{'sstable_compression': 'LZ4Compressor', 'lz4_compressor_type': 'high'};");
"{'class': 'LZ4Compressor', 'lz4_compressor_type': 'high'};");
ColumnFamilyStore store = flushTwice();
// Should flush as LZ4 "fast" mode
@ -155,7 +155,7 @@ public class CQLCompressionTest extends CQLTester
@Test
public void zstdFlushTest() throws Throwable
{
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'ZstdCompressor'};");
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'ZstdCompressor'};");
ColumnFamilyStore store = flushTwice();
// Should flush as LZ4
@ -177,7 +177,7 @@ public class CQLCompressionTest extends CQLTester
@Test
public void deflateFlushTest() throws Throwable
{
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'DeflateCompressor'};");
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'DeflateCompressor'};");
ColumnFamilyStore store = flushTwice();
// Should flush as LZ4
@ -200,7 +200,7 @@ public class CQLCompressionTest extends CQLTester
public void useNoCompressorOnFlushTest() throws Throwable
{
DatabaseDescriptor.setFlushCompression(Config.FlushCompression.none);
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'LZ4Compressor'};");
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'LZ4Compressor'};");
ColumnFamilyStore store = flushTwice();
// Should flush as Noop compressor
@ -224,7 +224,7 @@ public class CQLCompressionTest extends CQLTester
{
DatabaseDescriptor.setFlushCompression(Config.FlushCompression.table);
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'ZstdCompressor'};");
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'ZstdCompressor'};");
ColumnFamilyStore store = flushTwice();
// Should flush as Zstd
@ -237,7 +237,7 @@ public class CQLCompressionTest extends CQLTester
@Test
public void zstdTableFlushTest() throws Throwable
{
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'sstable_compression': 'ZstdCompressor'};");
createTable("CREATE TABLE %s (k text PRIMARY KEY, v text) WITH compression = {'class': 'ZstdCompressor'};");
ColumnFamilyStore store = flushTwice();
// Should flush as LZ4

2
tools/cqlstress-example.yaml
View File

@ -44,7 +44,7 @@ table_definition: |
PRIMARY KEY((name,choice), date, address, dbl, lval, ival, uid)
)
WITH compaction = { 'class':'LeveledCompactionStrategy' }
# AND compression = { 'sstable_compression' : '' }
# AND compression = { 'class' : 'LZ4Compressor' }
# AND comment='A table of many types to test wide rows'
#

4
tools/stress/src/org/apache/cassandra/stress/settings/SettingsSchema.java
View File

@ -140,7 +140,7 @@ public class SettingsSchema implements Serializable
//Compression
b.append(") WITH compression = {");
if (compression != null)
b.append("'sstable_compression' : '").append(compression).append("'");
b.append("'class' : '").append(compression).append("'");
b.append("}");
@ -181,7 +181,7 @@ public class SettingsSchema implements Serializable
//Compression
b.append(") WITH compression = {");
if (compression != null)
b.append("'sstable_compression' : '").append(compression).append("'");
b.append("'class' : '").append(compression).append("'");
b.append("}");