From dece96f21dbdbbb3176d6544d72bef3d44571dc6 Mon Sep 17 00:00:00 2001 From: Jacek Lewandowski Date: Wed, 22 Nov 2023 12:16:57 +0100 Subject: [PATCH] Fix filtering system ks sstables for relocation on startup Patch by Jacek Lewandowski; reviewed by Benjamin Lerer for CASSANDRA-18963 --- CHANGES.txt | 1 + .../cassandra/service/CassandraDaemon.java | 6 +- .../test/SystemKeyspacesDataLocationTest.java | 88 +++++++++++++++++++ 3 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/SystemKeyspacesDataLocationTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 6ac97d7a99..4fedf091a2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0.12 + * Fix filtering system ks sstables for relocation on startup (CASSANDRA-18963) * Remove completed coordinator sessions (CASSANDRA-18903) * Make StartupConnectivityChecker only run a connectivity check if there are no nodes which are running a version prior to Cassandra 4 (CASSANDRA-18968) * Retrieve keyspaces metadata and schema version concistently in DescribeStatement (CASSANDRA-18921) diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 6f74452ba4..6f12abf835 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; - import javax.management.ObjectName; import javax.management.StandardMBean; import javax.management.remote.JMXConnectorServer; @@ -50,7 +49,6 @@ import com.codahale.metrics.jvm.BufferPoolMetricSet; import com.codahale.metrics.jvm.FileDescriptorRatioGauge; import com.codahale.metrics.jvm.GarbageCollectorMetricSet; import com.codahale.metrics.jvm.MemoryUsageGaugeSet; - import org.apache.cassandra.audit.AuditLogManager; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; @@ -540,9 +538,7 @@ public class CassandraDaemon try (Stream keyspaceChildren = Files.list(keyspaceDirectory)) { Path[] tableDirectories = keyspaceChildren.filter(Files::isDirectory) - .filter(p -> !SystemKeyspace.TABLES_SPLIT_ACROSS_MULTIPLE_DISKS - .contains(p.getFileName() - .toString())) + .filter(p -> SystemKeyspace.TABLES_SPLIT_ACROSS_MULTIPLE_DISKS.stream().noneMatch(t -> p.getFileName().toString().startsWith(t + '-'))) .toArray(Path[]::new); for (Path tableDirectory : tableDirectories) diff --git a/test/distributed/org/apache/cassandra/distributed/test/SystemKeyspacesDataLocationTest.java b/test/distributed/org/apache/cassandra/distributed/test/SystemKeyspacesDataLocationTest.java new file mode 100644 index 0000000000..2b31a25a2f --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/SystemKeyspacesDataLocationTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import org.junit.Test; + +import org.apache.cassandra.cql3.QueryHandler; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; + +import static com.datastax.driver.core.ParseUtils.doubleQuote; +import static org.assertj.core.api.Assertions.assertThat; + +public class SystemKeyspacesDataLocationTest extends TestBaseImpl +{ + @Test + public void preparedStatementsShouldNotBeMovedOnStartupTest() throws IOException, ExecutionException, InterruptedException + { + try (Cluster cluster = Cluster.build(1).withDataDirCount(3).start()) + { + // to test the behaviour we need have sstables of prepared statements in more than one data directory + cluster.get(1).runOnInstance(() -> { + ColumnFamilyStore preparedStatementsCFS = ColumnFamilyStore.getIfExists(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PREPARED_STATEMENTS); + preparedStatementsCFS.disableAutoCompaction(); + createSSTablesForPreparedStatementsTable(preparedStatementsCFS); + }); + + Set preparedStatementsDataLocationsBefore = getPreparedStatementsDataLocations(cluster.get(1)); + assertThat(preparedStatementsDataLocationsBefore).hasSizeGreaterThan(1); + cluster.get(1).shutdown().get(); + + // we expect that for prepared statements, sstables will not be moved to the first data directory on startup + cluster.get(1).startup(); + Set preparedStatementsDataLocationsAfter = getPreparedStatementsDataLocations(cluster.get(1)); + assertThat(preparedStatementsDataLocationsAfter).isEqualTo(preparedStatementsDataLocationsBefore); + } + } + + private static Set getPreparedStatementsDataLocations(IInvokableInstance i) + { + return i.callOnInstance(() -> ColumnFamilyStore.getIfExists(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PREPARED_STATEMENTS) + .getLiveSSTables().stream() + // sstables are located at // so we need to go up two directories to get the data location + .map(sstr -> sstr.descriptor.directory.getParentFile().getParentFile().toString()) + .collect(Collectors.toSet())); + } + + private static void createSSTablesForPreparedStatementsTable(ColumnFamilyStore preparedStatementsCFS) + { + for (String keyspaceName : SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES) + { + for (TableMetadata tableMetadata : Schema.instance.getKeyspaceMetadata(keyspaceName).tables) + { + String query = String.format("select * from %s.%s", keyspaceName, doubleQuote(tableMetadata.name)); + QueryHandler.Prepared prepared = QueryProcessor.prepareInternal(query); + QueryProcessor.storePreparedStatement(query, keyspaceName, prepared); + preparedStatementsCFS.forceBlockingFlush(); + } + } + } +}