Merge branch 'cassandra-5.0' into trunk

CHANGES.txt

@@ -3,6 +3,7 @@
  * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781)
  * Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787)
 Merged from 5.0:
+ * Fix vector type to support DDM's mask_default function (CASSANDRA-18889)
  * Remove unnecessary reporter-config3 dependency (CASSANDRA-18907)
  * Remove support for empty values on the vector data type (CASSANDRA-18876)
  * Upgrade Dropwizard Metrics to 4.2.19 (CASSANDRA-14667)

doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql

@@ -2,7 +2,7 @@ CREATE FUNCTION redact(input text)
    CALLED ON NULL INPUT
    RETURNS text
    LANGUAGE java
-   AS 'return "redacted";
+   AS 'return "redacted";';
 
 CREATE TABLE patients (
    id timeuuid PRIMARY KEY,

doc/modules/cassandra/partials/masking_functions.adoc

@@ -10,7 +10,12 @@ Examples:
 
 `mask_null(123)` -> `null`
 
-| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default value of the same type. This will be `\***\***` for text values, zero for numeric values, `false` for booleans, etc.
+| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default value of the same type.
+This will be `\***\***` for text values, zero for numeric values, `false` for booleans, etc.
+
+Variable-length multivalued types such as lists, sets and maps are masked as empty collections.
+
+Fixed-length multivalued types such as tuples, UDTs and vectors are masked by replacing each of their values by the default masking value of the value type.
 
 Examples:
 
@@ -18,6 +23,10 @@ Examples:
 
 `mask_default(123)` -> `0`
 
+`mask_default((list<int>) [1, 2, 3])` -> `[]`
+
+`mask_default((vector<int, 3>) [1, 2, 3])` -> `[0, 0, 0]`
+
 | `mask_replace(value, replacement])` | Replaces the first argument by the replacement value on the second argument. The replacement value needs to have the same type as the replaced value.
 
 Examples:

src/java/org/apache/cassandra/cql3/CQL3Type.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.cql3;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -566,6 +567,21 @@ public interface CQL3Type
             return sb.toString();
         }
 
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Vector vector = (Vector) o;
+            return Objects.equals(type, vector.type);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(type);
+        }
+
         @Override
         public String toString()
         {

src/java/org/apache/cassandra/db/marshal/VectorType.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.db.marshal;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -167,6 +168,11 @@ public final class VectorType<T> extends AbstractType<List<T>>
         return array;
     }
 
+    public ByteBuffer decompose(T... values)
+    {
+        return decompose(Arrays.asList(values));
+    }
+
     public ByteBuffer decomposeAsFloat(float[] value)
     {
         return decomposeAsFloat(ByteBufferAccessor.instance, value);
@@ -367,6 +373,13 @@ public final class VectorType<T> extends AbstractType<List<T>>
         throw new MarshalException("Invalid empty vector value");
     }
 
+    @Override
+    public ByteBuffer getMaskedValue()
+    {
+        List<ByteBuffer> values = Collections.nCopies(dimension, elementType.getMaskedValue());
+        return serializer.serializeRaw(values, ByteBufferAccessor.instance);
+    }
+
     public abstract class VectorSerializer extends TypeSerializer<List<T>>
     {
         public abstract <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR);

test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java

@@ -25,6 +25,9 @@ import java.util.List;
 
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.VectorType;
+
 /**
  * {@link ColumnMaskQueryTester} for {@link DefaultMaskingFunction}.
  */
@@ -39,7 +42,11 @@ public class ColumnMaskQueryWithDefaultTest extends ColumnMaskQueryTester
             options.add(new Object[]{ order, "DEFAULT", "text", "abc", "****" });
             options.add(new Object[]{ order, "DEFAULT", "int", 123, 0 });
             options.add(new Object[]{ order, "mask_default()", "text", "abc", "****" });
-            options.add(new Object[]{ order, "mask_default()", "int", 123, 0, });
+            options.add(new Object[]{ order, "mask_default()", "int", 123, 0 });
+            // TODO: the driver version that we use doesn't support vectors, so we have to use raw values by now
+            options.add(new Object[]{ order, "mask_default()", "vector<int, 2>",
+                                      VectorType.getInstance(Int32Type.instance, 2).decompose(1, 2),
+                                      VectorType.getInstance(Int32Type.instance, 2).decompose(0, 0) });
         }
         return options;
     }

test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java

@@ -26,6 +26,9 @@ import java.util.List;
 
 import org.junit.runners.Parameterized;
 
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.VectorType;
+
 /**
  * {@link ColumnMaskQueryTester} for {@link ReplaceMaskingFunction}.
  */
@@ -43,6 +46,10 @@ public class ColumnMaskQueryWithReplaceTest extends ColumnMaskQueryTester
             options.add(new Object[]{ order, "mask_replace(0)", "int", 123, 0 });
             options.add(new Object[]{ order, "mask_replace(0)", "bigint", 123L, 0L });
             options.add(new Object[]{ order, "mask_replace(0)", "varint", BigInteger.valueOf(123), BigInteger.ZERO });
+            // TODO: the driver version that we use doesn't support vectors, so we have to use raw values by now
+            options.add(new Object[]{ order, "mask_replace([0, 0])", "vector<int, 2>",
+                                      VectorType.getInstance(Int32Type.instance, 2).decompose(1, 2),
+                                      VectorType.getInstance(Int32Type.instance, 2).decompose(0, 0) });
         }
         return options;
     }

test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java

@@ -108,6 +108,22 @@ public class ColumnMaskTest extends ColumnMaskTester
         assertTableColumnsAreNotMasked("v");
     }
 
+    @Test
+    public void testVectors() throws Throwable
+    {
+        // Create table with mask
+        String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 3> MASKED WITH DEFAULT)");
+        assertColumnIsMasked(table, "v", "mask_default", emptyList(), emptyList());
+
+        // Alter column mask
+        alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_null()");
+        assertColumnIsMasked(table, "v", "mask_null", emptyList(), emptyList());
+
+        // Drop mask
+        alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+        assertTableColumnsAreNotMasked("v");
+    }
+
     @Test
     public void testAlterTableAddMaskingToNonExistingColumn() throws Throwable
     {

test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.Duration;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.FloatType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.ListType;
 import org.apache.cassandra.db.marshal.MapType;
@@ -42,6 +43,7 @@ import org.apache.cassandra.db.marshal.SetType;
 import org.apache.cassandra.db.marshal.TupleType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.marshal.VectorType;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.serializers.SimpleDateSerializer;
@@ -159,6 +161,20 @@ public abstract class MaskingFunctionTester extends CQLTester
         testMaskingOnNotKeyColumns(MapType.getInstance(Int32Type.instance, Int32Type.instance, true).asCQL3Type(), values);
     }
 
+    /**
+     * Tests the native masking function for vectors.
+     */
+    @Test
+    public void testMaskingOnVector() throws Throwable
+    {
+        testMaskingOnAllColumns(VectorType.getInstance(Int32Type.instance, 2).asCQL3Type(),
+                                vector(1, 10), vector(2, 20));
+        testMaskingOnAllColumns(VectorType.getInstance(FloatType.instance, 2).asCQL3Type(),
+                                vector(1.1f, 10.1f), vector(2.2f, 20.2f));
+        testMaskingOnAllColumns(VectorType.getInstance(UTF8Type.instance, 2).asCQL3Type(),
+                                vector("a1", "a2"), vector("b1", "b2"));
+    }
+
     /**
      * Tests the native masking function for tuples.
      */

test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.cql3.functions.masking;
 
 import java.math.BigInteger;
+import java.nio.ByteBuffer;
 
 import org.junit.Test;
 
@@ -34,7 +35,7 @@ import static java.lang.String.format;
 public class ReplaceMaskingFunctionTest extends MaskingFunctionTester
 {
     @Override
-    protected void testMaskingOnColumn(String name, CQL3Type type, Object value) throws Throwable
+    protected void testMaskingOnColumn(String name, CQL3Type type, Object value)
     {
         // null replacement argument
         assertRows(execute(format("SELECT mask_replace(%s, ?) FROM %%s", name), (Object) null),
@@ -42,7 +43,7 @@ public class ReplaceMaskingFunctionTest extends MaskingFunctionTester
 
         // not-null replacement argument
         AbstractType<?> t = type.getType();
-        Object replacementValue = t.compose(t.getMaskedValue());
+        ByteBuffer replacementValue = t.getMaskedValue();
         String query = format("SELECT mask_replace(%s, ?) FROM %%s", name);
         assertRows(execute(query, replacementValue), row(replacementValue));
     }

test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java

@@ -101,7 +101,21 @@ public class AbstractTypeTest
     // TODO
     // isCompatibleWith/isValueCompatibleWith/isSerializationCompatibleWith,
     // withUpdatedUserType/expandUserTypes/referencesDuration - types that recursive check types
-    // getMaskedValue
+
+    @Test
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public void maskedValue()
+    {
+        qt().forAll(genBuilder().withoutTypeKinds(COMPOSITE, DYNAMIC_COMPOSITE).build())
+            .checkAssert(type -> {
+                ByteBuffer maskedValue = type.getMaskedValue();
+                type.validate(maskedValue);
+
+                Object composed = type.compose(maskedValue);
+                ByteBuffer decomposed = ((AbstractType) type).decompose(composed);
+                assertThat(decomposed).isEqualTo(maskedValue);
+            });
+    }
 
     @Test
     public void empty()