natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix vector type to support DDM's mask_default function 

patch by Andrés de la Peña; reviewed by Berenguer Blasi and Maxwell Guo for CASSANDRA-18889
This commit is contained in:
Andrés de la Peña 2023-09-27 21:56:15 +01:00
parent 8b941a6bdc
commit 3b9881bfa6
11 changed files with 106 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.0-alpha2
* Fix vector type to support DDM's mask_default function (CASSANDRA-18889)
* Remove unnecessary reporter-config3 dependency (CASSANDRA-18907)
* Remove support for empty values on the vector data type (CASSANDRA-18876)
* Upgrade Dropwizard Metrics to 4.2.19 (CASSANDRA-14667)

2
doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
View File

@ -2,7 +2,7 @@ CREATE FUNCTION redact(input text)
CALLED ON NULL INPUT
RETURNS text
LANGUAGE java
AS 'return "redacted";
AS 'return "redacted";';
CREATE TABLE patients (
id timeuuid PRIMARY KEY,

11
doc/modules/cassandra/partials/masking_functions.adoc
View File

@ -10,7 +10,12 @@ Examples:
`mask_null(123)` -> `null`
| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default value of the same type. This will be `\***\***` for text values, zero for numeric values, `false` for booleans, etc.
| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default value of the same type.
This will be `\***\***` for text values, zero for numeric values, `false` for booleans, etc.
Variable-length multivalued types such as lists, sets and maps are masked as empty collections.
Fixed-length multivalued types such as tuples, UDTs and vectors are masked by replacing each of their values by the default masking value of the value type.
Examples:
@ -18,6 +23,10 @@ Examples:
`mask_default(123)` -> `0`
`mask_default((list<int>) [1, 2, 3])` -> `[]`
`mask_default((vector<int, 3>) [1, 2, 3])` -> `[0, 0, 0]`
| `mask_replace(value, replacement])` | Replaces the first argument by the replacement value on the second argument. The replacement value needs to have the same type as the replaced value.
Examples:

16
src/java/org/apache/cassandra/cql3/CQL3Type.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -566,6 +567,21 @@ public interface CQL3Type
return sb.toString();
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Vector vector = (Vector) o;
return Objects.equals(type, vector.type);
}
@Override
public int hashCode()
{
return Objects.hash(type);
}
@Override
public String toString()
{

13
src/java/org/apache/cassandra/db/marshal/VectorType.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@ -167,6 +168,11 @@ public final class VectorType<T> extends AbstractType<List<T>>
return array;
}
public ByteBuffer decompose(T... values)
{
return decompose(Arrays.asList(values));
}
public ByteBuffer decomposeAsFloat(float[] value)
{
return decomposeAsFloat(ByteBufferAccessor.instance, value);
@ -367,6 +373,13 @@ public final class VectorType<T> extends AbstractType<List<T>>
throw new MarshalException("Invalid empty vector value");
}
@Override
public ByteBuffer getMaskedValue()
{
List<ByteBuffer> values = Collections.nCopies(dimension, elementType.getMaskedValue());
return serializer.serializeRaw(values, ByteBufferAccessor.instance);
}
public abstract class VectorSerializer extends TypeSerializer<List<T>>
{
public abstract <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR);

9
test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
View File

@ -25,6 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.VectorType;
/**
* {@link ColumnMaskQueryTester} for {@link DefaultMaskingFunction}.
*/
@ -39,7 +42,11 @@ public class ColumnMaskQueryWithDefaultTest extends ColumnMaskQueryTester
options.add(new Object[]{ order, "DEFAULT", "text", "abc", "****" });
options.add(new Object[]{ order, "DEFAULT", "int", 123, 0 });
options.add(new Object[]{ order, "mask_default()", "text", "abc", "****" });
options.add(new Object[]{ order, "mask_default()", "int", 123, 0, });
options.add(new Object[]{ order, "mask_default()", "int", 123, 0 });
// TODO: the driver version that we use doesn't support vectors, so we have to use raw values by now
options.add(new Object[]{ order, "mask_default()", "vector<int, 2>",
VectorType.getInstance(Int32Type.instance, 2).decompose(1, 2),
VectorType.getInstance(Int32Type.instance, 2).decompose(0, 0) });
}
return options;
}

7
test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
View File

@ -26,6 +26,9 @@ import java.util.List;
import org.junit.runners.Parameterized;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.VectorType;
/**
* {@link ColumnMaskQueryTester} for {@link ReplaceMaskingFunction}.
*/
@ -43,6 +46,10 @@ public class ColumnMaskQueryWithReplaceTest extends ColumnMaskQueryTester
options.add(new Object[]{ order, "mask_replace(0)", "int", 123, 0 });
options.add(new Object[]{ order, "mask_replace(0)", "bigint", 123L, 0L });
options.add(new Object[]{ order, "mask_replace(0)", "varint", BigInteger.valueOf(123), BigInteger.ZERO });
// TODO: the driver version that we use doesn't support vectors, so we have to use raw values by now
options.add(new Object[]{ order, "mask_replace([0, 0])", "vector<int, 2>",
VectorType.getInstance(Int32Type.instance, 2).decompose(1, 2),
VectorType.getInstance(Int32Type.instance, 2).decompose(0, 0) });
}
return options;
}

16
test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
View File

@ -108,6 +108,22 @@ public class ColumnMaskTest extends ColumnMaskTester
assertTableColumnsAreNotMasked("v");
}
@Test
public void testVectors() throws Throwable
{
// Create table with mask
String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 3> MASKED WITH DEFAULT)");
assertColumnIsMasked(table, "v", "mask_default", emptyList(), emptyList());
// Alter column mask
alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_null()");
assertColumnIsMasked(table, "v", "mask_null", emptyList(), emptyList());
// Drop mask
alterTable("ALTER TABLE %s ALTER v DROP MASKED");
assertTableColumnsAreNotMasked("v");
}
@Test
public void testAlterTableAddMaskingToNonExistingColumn() throws Throwable
{

16
test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
View File

@ -35,6 +35,7 @@ import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.FloatType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.db.marshal.MapType;
@ -42,6 +43,7 @@ import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.db.marshal.VectorType;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.serializers.SimpleDateSerializer;
@ -159,6 +161,20 @@ public abstract class MaskingFunctionTester extends CQLTester
testMaskingOnNotKeyColumns(MapType.getInstance(Int32Type.instance, Int32Type.instance, true).asCQL3Type(), values);
}
/**
* Tests the native masking function for vectors.
*/
@Test
public void testMaskingOnVector() throws Throwable
{
testMaskingOnAllColumns(VectorType.getInstance(Int32Type.instance, 2).asCQL3Type(),
vector(1, 10), vector(2, 20));
testMaskingOnAllColumns(VectorType.getInstance(FloatType.instance, 2).asCQL3Type(),
vector(1.1f, 10.1f), vector(2.2f, 20.2f));
testMaskingOnAllColumns(VectorType.getInstance(UTF8Type.instance, 2).asCQL3Type(),
vector("a1", "a2"), vector("b1", "b2"));
}
/**
* Tests the native masking function for tuples.
*/

5
test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
View File

@ -19,6 +19,7 @@
package org.apache.cassandra.cql3.functions.masking;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import org.junit.Test;
@ -34,7 +35,7 @@ import static java.lang.String.format;
public class ReplaceMaskingFunctionTest extends MaskingFunctionTester
{
@Override
protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
protected void testMaskingOnColumn(String name, CQL3Type type, Object value)
{
// null replacement argument
assertRows(execute(format("SELECT mask_replace(%s, ?) FROM %%s", name), (Object) null),
@ -42,7 +43,7 @@ public class ReplaceMaskingFunctionTest extends MaskingFunctionTester
// not-null replacement argument
AbstractType<?> t = type.getType();
Object replacementValue = t.compose(t.getMaskedValue());
ByteBuffer replacementValue = t.getMaskedValue();
String query = format("SELECT mask_replace(%s, ?) FROM %%s", name);
assertRows(execute(query, replacementValue), row(replacementValue));
}

16
test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
View File

@ -101,7 +101,21 @@ public class AbstractTypeTest
// TODO
// isCompatibleWith/isValueCompatibleWith/isSerializationCompatibleWith,
// withUpdatedUserType/expandUserTypes/referencesDuration - types that recursive check types
// getMaskedValue
@Test
@SuppressWarnings({"unchecked", "rawtypes"})
public void maskedValue()
{
qt().forAll(genBuilder().withoutTypeKinds(COMPOSITE, DYNAMIC_COMPOSITE).build())
.checkAssert(type -> {
ByteBuffer maskedValue = type.getMaskedValue();
type.validate(maskedValue);
Object composed = type.compose(maskedValue);
ByteBuffer decomposed = ((AbstractType) type).decompose(composed);
assertThat(decomposed).isEqualTo(maskedValue);
});
}
@Test
public void empty()