natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add guardrail for vector dimensions 

patch by Andrés de la Peña; reviewed by Brandon Williams and Maxwell Guo for CASSANDRA-18730
This commit is contained in:
Andrés de la Peña 2023-08-07 12:46:08 +01:00
parent ada67cb2a4
commit 7482d88f48
14 changed files with 350 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.0
* Add guardrail for vector dimensions (CASSANDRA-18730)
* Upgraded Netty to 4.1.96, added BouncyCastle dependency (CASSANDRA-17992)
* Fix for (unsupported) big endian unaligned architecture, eg s390x (CASSANDRA-17723)
* CIDR filtering authorizer (CASSANDRA-18592)

1
NEWS.txt
View File

@ -116,6 +116,7 @@ New features
- Column value size
- Partition size
- Partition tombstones
- Vector dimensions
- It is possible to list ephemeral snapshots by nodetool listsnaphots command when flag "-e" is specified.
- Added a new flag to `nodetool profileload` and JMX endpoint to set up recurring profile load generation on specified
intervals (see CASSANDRA-17821)

5
conf/cassandra.yaml
View File

@ -1989,6 +1989,11 @@ drop_compact_storage_enabled: false
# fields_per_udt_warn_threshold: -1
# fields_per_udt_fail_threshold: -1
#
# Guardrail to warn or fail when creating a vector column with more dimensions than threshold.
# Default -1 to disable.
# vector_dimensions_warn_threshold: -1
# vector_dimensions_fail_threshold: -1
#
# Guardrail to indicate whether or not users are allowed to use ALTER TABLE commands to make column changes to tables
# alter_table_enabled: true
#

2
src/java/org/apache/cassandra/config/Config.java
View File

@ -892,6 +892,8 @@ public class Config
public volatile int items_per_collection_fail_threshold = -1;
public volatile int fields_per_udt_warn_threshold = -1;
public volatile int fields_per_udt_fail_threshold = -1;
public volatile int vector_dimensions_warn_threshold = -1;
public volatile int vector_dimensions_fail_threshold = -1;
public volatile int data_disk_usage_percentage_warn_threshold = -1;
public volatile int data_disk_usage_percentage_fail_threshold = -1;
public volatile DataStorageSpec.LongBytesBound data_disk_usage_max_disk_size = null;

26
src/java/org/apache/cassandra/config/GuardrailsOptions.java
View File

@ -82,6 +82,7 @@ public class GuardrailsOptions implements GuardrailsConfig
validateSizeThreshold(config.collection_size_warn_threshold, config.collection_size_fail_threshold, false, "collection_size");
validateMaxIntThreshold(config.items_per_collection_warn_threshold, config.items_per_collection_fail_threshold, "items_per_collection");
validateMaxIntThreshold(config.fields_per_udt_warn_threshold, config.fields_per_udt_fail_threshold, "fields_per_udt");
validateMaxIntThreshold(config.vector_dimensions_warn_threshold, config.vector_dimensions_fail_threshold, "vector_dimensions");
validatePercentageThreshold(config.data_disk_usage_percentage_warn_threshold, config.data_disk_usage_percentage_fail_threshold, "data_disk_usage_percentage");
validateDataDiskUsageMaxDiskSize(config.data_disk_usage_max_disk_size);
validateMinRFThreshold(config.minimum_replication_factor_warn_threshold, config.minimum_replication_factor_fail_threshold);
@ -697,6 +698,31 @@ public class GuardrailsOptions implements GuardrailsConfig
x -> config.fields_per_udt_fail_threshold = x);
}
@Override
public int getVectorDimensionsWarnThreshold()
{
return config.vector_dimensions_warn_threshold;
}
@Override
public int getVectorDimensionsFailThreshold()
{
return config.vector_dimensions_fail_threshold;
}
public void setVectorDimensionsThreshold(int warn, int fail)
{
validateMaxIntThreshold(warn, fail, "vector_dimensions");
updatePropertyWithLogging("vector_dimensions_warn_threshold",
warn,
() -> config.vector_dimensions_warn_threshold,
x -> config.vector_dimensions_warn_threshold = x);
updatePropertyWithLogging("vector_dimensions_fail_threshold",
fail,
() -> config.vector_dimensions_fail_threshold,
x -> config.vector_dimensions_fail_threshold = x);
}
public int getDataDiskUsagePercentageWarnThreshold()
{
return config.data_disk_usage_percentage_warn_threshold;

53
src/java/org/apache/cassandra/cql3/CQL3Type.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.marshal.CollectionType.Kind;
import org.apache.cassandra.exceptions.InvalidRequestException;
@ -35,6 +36,7 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.utils.ByteBufferUtil;
import static java.util.stream.Collectors.toList;
@ -53,6 +55,11 @@ public interface CQL3Type
return false;
}
default boolean isVector()
{
return false;
}
public AbstractType<?> getType();
/**
@ -528,6 +535,11 @@ public interface CQL3Type
this.type = VectorType.getInstance(elementType, dimensions);
}
public boolean isVector()
{
return true;
}
@Override
public VectorType<?> getType()
{
@ -622,6 +634,8 @@ public interface CQL3Type
throw new InvalidRequestException(message);
}
public abstract void validate(ClientState state, String name);
public CQL3Type prepare(String keyspace)
{
KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
@ -687,6 +701,16 @@ public interface CQL3Type
this.type = type;
}
@Override
public void validate(ClientState state, String name)
{
if (type.isVector())
{
int dimensions = ((Vector) type).getType().dimension;
Guardrails.vectorDimensions.guard(dimensions, name, false, state);
}
}
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
{
return type;
@ -754,6 +778,16 @@ public interface CQL3Type
return true;
}
@Override
public void validate(ClientState state, String name)
{
if (keys != null)
keys.validate(state, name);
if (values != null)
values.validate(state, name);
}
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
{
return prepare(keyspace, udts, false);
@ -861,6 +895,12 @@ public interface CQL3Type
return this;
}
@Override
public void validate(ClientState state, String name)
{
Guardrails.vectorDimensions.guard(dimension, name, false, state);
}
@Override
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
{
@ -896,6 +936,12 @@ public interface CQL3Type
return new RawUT(name, true);
}
@Override
public void validate(ClientState state, String name)
{
// nothing to do here
}
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
{
if (name.hasKeyspace())
@ -969,6 +1015,13 @@ public interface CQL3Type
return this;
}
@Override
public void validate(ClientState state, String name)
{
for (CQL3Type.Raw t : types)
t.validate(state, name);
}
public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException
{
List<AbstractType<?>> ts = new ArrayList<>(types.size());

7
src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
View File

@ -267,6 +267,13 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
this.ifColumnNotExists = ifColumnNotExists;
}
@Override
public void validate(ClientState state)
{
super.validate(state);
newColumns.forEach(c -> c.type.validate(state, "Column " + c.name));
}
public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
{
Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);

1
src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java
View File

@ -147,6 +147,7 @@ public abstract class AlterTypeStatement extends AlterSchemaStatement
}
Guardrails.fieldsPerUDT.guard(userType.size() + 1, userType.getNameAsString(), false, state);
type.validate(state, "Field " + fieldName);
List<FieldIdentifier> fieldNames = new ArrayList<>(userType.fieldNames()); fieldNames.add(fieldName);
List<AbstractType<?>> fieldTypes = new ArrayList<>(userType.fieldTypes()); fieldTypes.add(fieldType);

15
src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
View File

@ -152,12 +152,7 @@ public final class CreateTableStatement extends AlterSchemaStatement
validateDefaultTimeToLive(attrs.asNewTableParams());
// Verify that dynamic data masking is enabled if there are masked columns
for (ColumnProperties.Raw raw : rawColumns.values())
{
if (raw.rawMask != null)
ColumnMask.ensureEnabled();
}
rawColumns.forEach((name, raw) -> raw.validate(state, name));
}
SchemaChange schemaChangeEvent(KeyspacesDiff diff)
@ -604,6 +599,14 @@ public final class CreateTableStatement extends AlterSchemaStatement
this.rawMask = rawMask;
}
public void validate(ClientState state, ColumnIdentifier name)
{
rawType.validate(state, "Column " + name);
if (rawMask != null)
ColumnMask.ensureEnabled();
}
public ColumnProperties prepare(String keyspace, String table, ColumnIdentifier column, Types udts)
{
CQL3Type cqlType = rawType.prepare(keyspace, udts);

5
src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java
View File

@ -68,6 +68,11 @@ public final class CreateTypeStatement extends AlterSchemaStatement
super.validate(state);
Guardrails.fieldsPerUDT.guard(fieldNames.size(), typeName, false, state);
for (int i = 0; i < rawFieldTypes.size(); i++)
{
rawFieldTypes.get(i).validate(state, "Field " + fieldNames.get(i));
}
}
public Keyspaces apply(Keyspaces schema)

30
src/java/org/apache/cassandra/db/guardrails/Guardrails.java
View File

@ -385,6 +385,18 @@ public final class Guardrails implements GuardrailsMBean
: format("User types cannot have more than %s columns, but %s provided for user type %s.",
threshold, value, what));
/**
* Guardrail on the number of dimensions of vector columns.
*/
public static final MaxThreshold vectorDimensions =
new MaxThreshold("vector_dimensions",
null,
state -> CONFIG_PROVIDER.getOrCreate(state).getVectorDimensionsWarnThreshold(),
state -> CONFIG_PROVIDER.getOrCreate(state).getVectorDimensionsFailThreshold(),
(isWarning, what, value, threshold) ->
format("%s has a vector of %s dimensions, this exceeds the %s threshold of %s.",
what, value, isWarning ? "warning" : "failure", threshold));
/**
* Guardrail on the data disk usage on the local node, used by a periodic task to calculate and propagate that status.
* See {@link org.apache.cassandra.service.disk.usage.DiskUsageMonitor} and {@link DiskUsageBroadcaster}.
@ -1043,6 +1055,24 @@ public final class Guardrails implements GuardrailsMBean
DEFAULT_CONFIG.setFieldsPerUDTThreshold(warn, fail);
}
@Override
public int getVectorDimensionsWarnThreshold()
{
return DEFAULT_CONFIG.getVectorDimensionsWarnThreshold();
}
@Override
public int getVectorDimensionsFailThreshold()
{
return DEFAULT_CONFIG.getVectorDimensionsFailThreshold();
}
@Override
public void setVectorDimensionsThreshold(int warn, int fail)
{
DEFAULT_CONFIG.setVectorDimensionsThreshold(warn, fail);
}
@Override
public int getMaximumReplicationFactorWarnThreshold()
{

10
src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
View File

@ -304,6 +304,16 @@ public interface GuardrailsConfig
*/
int getFieldsPerUDTFailThreshold();
/**
* @return The threshold to warn when creating a vector with more dimensions than threshold.
*/
int getVectorDimensionsWarnThreshold();
/**
* @return The threshold to fail when creating a vector with more dimensions than threshold.
*/
int getVectorDimensionsFailThreshold();
/**
* @return The threshold to warn when local disk usage percentage exceeds that threshold.
* Allowed values are in the range {@code [1, 100]}, and -1 means disabled.

16
src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
View File

@ -597,6 +597,22 @@ public interface GuardrailsMBean
*/
void setFieldsPerUDTThreshold(int warn, int fail);
/**
* @return The threshold to warn when creating a vector with more dimensions than threshold.
*/
int getVectorDimensionsWarnThreshold();
/**
* @return The threshold to fail when creating a vector with more dimensions than threshold.
*/
int getVectorDimensionsFailThreshold();
/**
* @param warn The threshold to warn when creating a vector with more dimensions than threshold.
* @param fail The threshold to prevent creating a vector with more dimensions than threshold.
*/
void setVectorDimensionsThreshold(int warn, int fail);
/**
* @return The threshold to warn when local data disk usage percentage exceeds that threshold.
* Allowed values are in the range {@code [1, 100]}, and -1 means disabled.

184
test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java Normal file
View File

@ -0,0 +1,184 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.guardrails;
import java.util.function.Supplier;
import org.junit.Test;
import static java.lang.String.format;
/**
* Tests the guardrail for the number of dimensions of a vector, {@link Guardrails#vectorDimensions}.
*/
public class GuardrailVectorDimensionsTest extends ThresholdTester
{
private static final int WARN_THRESHOLD = 2;
private static final int FAIL_THRESHOLD = 4;
public GuardrailVectorDimensionsTest()
{
super(WARN_THRESHOLD,
FAIL_THRESHOLD,
Guardrails.vectorDimensions,
Guardrails::setVectorDimensionsThreshold,
Guardrails::getVectorDimensionsWarnThreshold,
Guardrails::getVectorDimensionsFailThreshold);
}
@Test
public void testCreateTable() throws Throwable
{
// different poisitions (partition key, clustering key, static column, regular column)
testCreateTable("CREATE TABLE %s (v vector<int, %%d> PRIMARY KEY)");
testCreateTable("CREATE TABLE %s (v vector<int, %%d>, c int, PRIMARY KEY(v, c))");
testCreateTable("CREATE TABLE %s (v vector<int, %%d>, c int, PRIMARY KEY((v, c)))");
testCreateTable("CREATE TABLE %s (k int, v vector<int, %%d>, PRIMARY KEY(k, v))");
testCreateTable("CREATE TABLE %s (k int, c int, v vector<int, %%d>, PRIMARY KEY(k, c, v))");
testCreateTable("CREATE TABLE %s (k int, c int, v vector<int, %%d> static, PRIMARY KEY(k, c))");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, %%d>)");
// multivalued data types
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v list<vector<int, %%d>>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v set<vector<int, %%d>>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<int, vector<int, %%d>>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<vector<int, %%d>, int>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v tuple<vector<int, %%d>, int>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v tuple<int, vector<int, %%d>>)");
// nested multivalued data types
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v set<frozen<list<vector<int, %%d>>>>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v list<frozen<set<vector<int, %%d>>>>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<int, frozen<set<vector<int, %%d>>>>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<frozen<set<vector<int, %%d>>>, int>)");
testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v tuple<frozen<tuple<vector<int, %%d>, int>>, int>)");
}
private void testCreateTable(String query) throws Throwable
{
testColumn(() -> format(query, createTableName()));
}
@Test
public void testCreateType() throws Throwable
{
testCreateType("CREATE TYPE %s (c int, v vector<int, %%d>)");
testCreateType("CREATE TYPE %s (c int, v list<vector<int, %%d>>)");
testCreateType("CREATE TYPE %s (c int, v set<vector<int, %%d>>)");
testCreateType("CREATE TYPE %s (c int, v map<int, vector<int, %%d>>)");
testCreateType("CREATE TYPE %s (c int, v map<vector<int, %%d>, int>)");
testCreateType("CREATE TYPE %s (c int, v tuple<vector<int, %%d>, int>)");
testCreateType("CREATE TYPE %s (c int, v tuple<int, vector<int, %%d>>)");
}
private void testCreateType(String query) throws Throwable
{
testField(() -> format(query, createTypeName()));
}
@Test
public void testAlterTable() throws Throwable
{
testAlterTable("ALTER TABLE %s ADD v vector<int, %%d>");
testAlterTable("ALTER TABLE %s ADD v list<vector<int, %%d>>");
testAlterTable("ALTER TABLE %s ADD v set<vector<int, %%d>>");
testAlterTable("ALTER TABLE %s ADD v map<int, vector<int, %%d>>");
testAlterTable("ALTER TABLE %s ADD v map<vector<int, %%d>, int>");
testAlterTable("ALTER TABLE %s ADD v tuple<vector<int, %%d>, int>");
testAlterTable("ALTER TABLE %s ADD v tuple<int, vector<int, %%d>>");
}
private void testAlterTable(String query) throws Throwable
{
testColumn(() -> {
createTable("CREATE TABLE %s (k int PRIMARY KEY)");
return format(query, currentTable());
});
}
@Test
public void testAlterType() throws Throwable
{
testAlterType("ALTER TYPE %s ADD v vector<int, %%d>");
testAlterType("ALTER TYPE %s ADD v list<vector<int, %%d>>");
testAlterType("ALTER TYPE %s ADD v set<vector<int, %%d>>");
testAlterType("ALTER TYPE %s ADD v map<int, vector<int, %%d>>");
testAlterType("ALTER TYPE %s ADD v map<vector<int, %%d>, int>");
testAlterType("ALTER TYPE %s ADD v tuple<vector<int, %%d>, int>");
testAlterType("ALTER TYPE %s ADD v tuple<int, vector<int, %%d>>");
}
private void testAlterType(String query) throws Throwable
{
testField(() -> {
String name = createType("CREATE TYPE %s (c int)");
return format(query, name);
});
}
@Test
public void testExcludedUsers() throws Throwable
{
testExcludedUsers(() -> format("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 1000>)", createTableName()),
() -> format("CREATE TYPE %s (c int, v vector<int, 1000>)", createTypeName()),
() -> format("ALTER TABLE %s ADD v vector<int, 1000>",
createTable("CREATE TABLE %s (k int PRIMARY KEY)")),
() -> format("ALTER TYPE %s ADD v vector<int, 1000>",
createType("CREATE TYPE %s (c int)")));
}
private void testColumn(Supplier<String> query) throws Throwable
{
testGuardrail(query, "Column v");
}
private void testField(Supplier<String> query) throws Throwable
{
testGuardrail(query, "Field v");
}
private void testGuardrail(Supplier<String> query, String element) throws Throwable
{
assertValid(query.get(), 1);
assertValid(query.get(), WARN_THRESHOLD);
assertWarns(query.get(), element, WARN_THRESHOLD + 1);
assertWarns(query.get(), element, FAIL_THRESHOLD);
assertFails(query.get(), element, FAIL_THRESHOLD + 1);
assertFails(query.get(), element, Integer.MAX_VALUE);
}
private void assertValid(String query, int dimensions) throws Throwable
{
super.assertValid(format(query, dimensions));
}
private void assertWarns(String query, String what, int dimensions) throws Throwable
{
assertWarns(format(query, dimensions),
format(what + " has a vector of %s dimensions, this exceeds the warning threshold of %s.",
dimensions, WARN_THRESHOLD));
}
private void assertFails(String query, String what, int dimensions) throws Throwable
{
assertFails(format(query, dimensions),
format(what + " has a vector of %s dimensions, this exceeds the failure threshold of %s.",
dimensions, FAIL_THRESHOLD));
}
}