Implementation of Transactional Cluster Metadata as described in CEP-21

An overview of the core components can be found in the included
TransactionalClusterMetadata.md

patch by Alex Petrov, Marcus Eriksson and Sam Tunnicliffe; reviewed by
Alex Petrov, Marcus Eriksson and Sam Tunnicliffe for CASSANDRA-18330

Co-authored-by: Marcus Eriksson <marcuse@apache.org>
Co-authored-by: Alex Petrov <oleksandr.petrov@gmail.com>
Co-authored-by: Sam Tunnicliffe <samt@apache.org>

.build/build-resolver.xml

@@ -59,7 +59,7 @@
             <remoterepo id="resolver-central" url="${artifact.remoteRepository.central}"/>
             <remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}"/>
             <!-- Snapshots are not allowed, but for feature branches they may be needed, so uncomment the below to allow snapshots to work -->
-            <!-- <remoterepo id="resolver-apache-snapshot" url="https://repository.apache.org/content/repositories/snapshots" releases="false" snapshots="true" updates="always" checksums="fail" /> -->
+            <remoterepo id="resolver-apache-snapshot" url="https://repository.apache.org/content/repositories/snapshots" releases="false" snapshots="true" updates="always" checksums="fail" />
         </resolver:remoterepos>
 
         <macrodef name="resolve">
@@ -267,6 +267,10 @@
 
         <delete file="${build.lib}/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" failonerror="false"/>
         <delete file="${build.dir.lib}/jars/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" failonerror="false"/>
+
+        <copy todir="${test.lib}/jars" quiet="false">
+            <file file="${build.lib}/harry-core-0.0.2-CASSANDRA-18768.jar"/>
+       </copy>
     </target>
 
     <target name="_resolver-dist-lib_get_files">

.build/cassandra-build-deps-template.xml

@@ -70,6 +70,10 @@
       <groupId>com.puppycrawl.tools</groupId>
       <artifactId>checkstyle</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava-testlib</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.quicktheories</groupId>
       <artifactId>quicktheories</artifactId>

.build/parent-pom-template.xml

@@ -330,6 +330,11 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava-testlib</artifactId>
+        <version>27.0-jre</version>
+      </dependency>
       <dependency>
         <groupId>com.google.jimfs</groupId>
         <artifactId>jimfs</artifactId>
@@ -511,8 +516,9 @@
       <dependency>
         <groupId>org.apache.cassandra</groupId>
         <artifactId>harry-core</artifactId>
-        <version>0.0.1</version>
+        <version>harry-core-0.0.2-CASSANDRA-18768</version>
-        <scope>test</scope>
+        <scope>system</scope>
+        <systemPath>${test.lib}/harry-core-0.0.2-CASSANDRA-18768.jar</systemPath>
       </dependency>
       <dependency>
         <groupId>org.reflections</groupId>

.gitignore

@@ -17,6 +17,7 @@ pylib/src/
 pylib/cqlshlib/serverversion.py
 !lib/cassandra-driver-internal-only-*.zip
 !lib/puresasl-*.zip
+!lib/harry-0.0.2-internal-20221121.14211-2.jar
 
 # C* debs
 build-stamp

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Transactional Cluster Metadata [CEP-21] (CASSANDRA-18330)
  * Add ELAPSED command to cqlsh (CASSANDRA-18861)
  * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781)
  * Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787)

NEWS.txt

@@ -71,11 +71,70 @@ using the provided 'sstableupgrade' tool.
 
 New features
 ------------
+    [The following is a placeholder, to be revised asap]
+    - CEP-21 Transactional Cluster Metadata introduces a distributed log for linearizing modifications to cluster
+    metadata. In the first instance, this encompasses cluster membership, token ownership and schema metadata. See
+    https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-21%3A+Transactional+Cluster+Metadata for more detail on
+    the motivation and design, and see "Upgrading" below for specific instructions on migrating clusters to this system.
+    More updates and documentation to follow.
     - New Guardrails added:
       - Whether bulk loading of SSTables is allowed.
 
 Upgrading
 ---------
+    [The following is a placeholder, to be revised asap]
+    - Upgrading to 5.1 is currently only supported from 4.0, 4.1 or 5.0. Clusters running earlier releases must first
+    upgrade to at least the latest 4.0 release. When a cluster is upgraded to 5.1 there is an additional step for
+    operators to perform. The upgrade is not considered complete until:
+      1. All UP nodes are running 5.1
+      2. The Cluster Metadata Service (CMS) has been enabled
+    The first step is just a regular upgrade, there are no changes to external APIs, SSTable formats to consider.
+    Step 2 requires an operator to run a new nodetool subcommand, intializecms, on one node in the cluster.
+      - > nodetool initializecms
+    Doing so creates the CMS with the local node as its only member. The initializecms command cannot be executed until
+    step 1 is complete and all nodes are running 5.1 as migrating metadata management over to the new TCM system
+    requires all nodes to be in agreement over the current state of the cluster. Essentially this means agreement on
+    schema and topology. Once the upgrade has started, but before initializecms is run, metadata-changing operations are
+    not permitted and if attempted from an upgraded node will be rejected.
+    Prohibited operations include:
+      - schema changes
+      - node replacement
+      - bootstrap
+      - decommission
+      - move
+      - assasinate
+    For the time being there is no mechanism in place to prevent these operations being executed by a node still running
+    a previous version, though this is planned before release. Any automation which can trigger these operations should
+    be disabled for the cluster prior to starting the upgrade.
+    Should any of the prohibited operations be executed (i.e. on a node that is still running a pre-5.1 version) before
+    the CMS migration, nodes which are DOWN or which have been upgraded will not process the metadata changes. However,
+    nodes still UP and running the old version will. This will eventually cause the migration to fail, as the cluster
+    will not be in agreement.
+      - > nodetool initializecms
+        Got mismatching cluster metadatas from [/x.x.x.x:7000] aborting migration
+        See 'nodetool help' or 'nodetool help <command>'.
+    If the initializecms command fails, it will indicate which nodes’ current metadata does not agree with the node
+    where the command was executed. To mitigate this situation, bring any mismatching nodes DOWN and rerun the
+    initializecms command with the additional —ignore flag.
+      - nodetool intializecms -ignore x.x.x.x
+    Once the command has run successfully the ignored nodes can be restarted but any metadata changes that they
+    accepted/and or applied whilst the cluster was in mixed mode will be lost. We plan to improve this before beta, but
+    in the meantime operators should ensure no schema or membership/ownership changes are performed during the upgrade.
+    Although the restrictions on metadata-changing operations will be removed as soon as the initial CMS migration is
+    complete, at that point the CMS will only contain a single member, which is not suitable for real clusters. To
+    modify the membership of the CMS a second nodetool subcommand, reconfigurecms, should be run. This command allows
+    the number of required number of CMS members to be specified for each datacenter. Consensus for metadata operations
+    by the CMS is obtained via Paxos, operating at SERIAL/QUORUM consistency, so the minimum safe size for the CMS is 3
+    nodes. In multi-DC deployments, the CMS members may be distributed across DCs to ensure resilience in the case of a
+    DC outage. It is not recommended to make every node a member of the CMS. Redundancy is the primary concern here, not
+    locality or latency, so between 3 and 7 nodes per DC depending on the size of the cluster should be the goal.
+    Deploying to a fresh cluster is more straightforward. As the cluster comes up, at least one node will automatically
+    nominate itself as the first CMS node. A simple election mechanism ensures that only one node will be chosen if
+    multiple peers attempt to nominate themselves. As soon as the election is complete, metadata modifications are
+    supported. Typically, this takes only a few seconds and is likely to complete before all nodes are fully UP. Nodes
+    which come up during or after an election will learn of the elected first CMS node and direct metadata updates to
+    it. It is important to remember that at the completion of the election, the CMS still only comprises a single
+    member. Just as in the upgrade case, operators should add further members as soon as possible.
 
 Deprecation
 -----------

build.xml

@@ -403,7 +403,7 @@
 
     <target name="realclean" depends="clean" description="Remove the entire build directory and all downloaded artifacts">
         <delete>
-          <fileset dir="${build.lib}" excludes="cassandra-driver-internal-only-*,puresasl-internal-only-*"/>
+          <fileset dir="${build.lib}" excludes="cassandra-driver-internal-only-*,puresasl-internal-only-*,harry-*"/>
         </delete>
         <delete dir="${build.dir}" />
         <delete dir="${doc.dir}/build" />

ci/harry_simulation.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -o errexit
+set -o nounset
+set -o pipefail
+
+rm -f build/test/lib/jars/guava-18.0.jar
+current_dir=$(dirname "$(readlink -f "$0")")
+
+common=(-Dstorage-config=$current_dir/../test/conf
+        -Djava.awt.headless=true
+        -ea
+        -XX:SoftRefLRUPolicyMSPerMB=0
+        -XX:HeapDumpPath=build/test
+        -Dcassandra.test.driver.connection_timeout_ms=10000
+        -Dcassandra.test.driver.read_timeout_ms=24000
+        -Dcassandra.memtable_row_overhead_computation_step=100
+        -Dcassandra.test.use_prepared=true
+        -Dcassandra.test.sstableformatdevelopment=true
+        -Djava.security.egd=file:/dev/urandom
+        -Dcassandra.testtag=.jdk11
+        -Dstorage-config=$current_dir/../test/conf
+        -Djava.awt.headless=true
+        -Dcassandra.keepBriefBrief=true
+        -Dcassandra.allow_simplestrategy=true
+        -Dcassandra.strict.runtime.checks=true
+        -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true
+        -Dcassandra.test.flush_local_schema_changes=false
+        -Dcassandra.test.messagingService.nonGracefulShutdown=true
+        -Dcassandra.use_nix_recursive_delete=true
+        -Dcie-cassandra.disable_schema_drop_log=true
+        -Dcassandra.ring_delay_ms=10000
+        -Dcassandra.tolerate_sstable_size=true
+        -Dcassandra.skip_sync=true
+        -Dcassandra.debugrefcount=false
+        -Dcassandra.test.simulator.determinismcheck=strict
+        -Dcassandra.test.simulator.print_asm=none
+        -Dlog4j2.disableJmx=true
+        -Dlog4j2.disable.jmx=true
+        -Dlog4j.shutdownHookEnabled=false
+        -Dcassandra.test.logConfigPath=$current_dir/../test/conf/log4j2-dtest-simulator.xml
+        -Dcassandra.test.logConfigProperty=log4j.configurationFile
+        -Dlog4j2.configurationFile=$current_dir/../test/conf/log4j2-dtest-simulator.xml
+        -javaagent:$current_dir/../lib/jamm-0.3.2.jar
+        -javaagent:$current_dir/../build/test/lib/jars/simulator-asm.jar
+        -Xbootclasspath/a:$current_dir/../build/test/lib/jars/simulator-bootstrap.jar
+        -XX:ActiveProcessorCount=4
+        -XX:-TieredCompilation
+        -XX:-BackgroundCompilation
+        -XX:CICompilerCount=1
+        -XX:Tier4CompileThreshold=1000
+        -XX:ReservedCodeCacheSize=256M
+        -Xmx16G
+        -Xmx4G
+        -Xss384k
+        --add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+        --add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+        --add-exports java.base/sun.nio.ch=ALL-UNNAMED
+        --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+        --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+        --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+        --add-exports java.sql/java.sql=ALL-UNNAMED
+        --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+        --add-opens java.base/java.lang.module=ALL-UNNAMED
+        --add-opens java.base/java.net=ALL-UNNAMED
+        --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+        --add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+        --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+        --add-opens java.base/jdk.internal.math=ALL-UNNAMED
+        --add-opens java.base/jdk.internal.module=ALL-UNNAMED
+        --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+        --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+        --add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED
+        --add-opens java.desktop/com.sun.beans.introspect=ALL-UNNAMED
+        -classpath $current_dir/../build/classes/main/:$current_dir/../build/test/classes/:$current_dir/../build/test/lib/jars/*:$current_dir/../build/lib/jars/*
+
+       )
+
+
+java "${common[@]}" org.apache.cassandra.simulator.test.HarrySimulatorTest

conf/cassandra.yaml

@@ -1201,16 +1201,16 @@ range_request_timeout: 10000ms
 # How long the coordinator should wait for writes to complete.
 # Lowest acceptable value is 10 ms.
 # Min unit: ms
-write_request_timeout: 2000ms
+write_request_timeout: 10000ms
 # How long the coordinator should wait for counter writes to complete.
 # Lowest acceptable value is 10 ms.
 # Min unit: ms
-counter_write_request_timeout: 5000ms
+counter_write_request_timeout: 1000ms
 # How long a coordinator should continue to retry a CAS operation
 # that contends with other proposals for the same row.
 # Lowest acceptable value is 10 ms.
 # Min unit: ms
-cas_contention_timeout: 1000ms
+cas_contention_timeout: 5000ms
 # How long the coordinator should wait for truncates to complete
 # (This can be much longer, because unless auto_snapshot is disabled
 # we need to flush first so we can snapshot before removing the data.)
@@ -2088,12 +2088,6 @@ drop_compact_storage_enabled: false
 #    enabled: false
 #    ownership_token: "sometoken" # (overriden by "CassandraOwnershipToken" system property)
 #    ownership_filename: ".cassandra_fs_ownership" # (overriden by "cassandra.fs_ownership_filename")
-# Prevents a node from starting if snitch's data center differs from previous data center.
-#  check_dc:
-#    enabled: true # (overriden by cassandra.ignore_dc system property)
-# Prevents a node from starting if snitch's rack differs from previous rack.
-#  check_rack:
-#    enabled: true # (overriden by cassandra.ignore_rack system property)
 # Enable this property to fail startup if the node is down for longer than gc_grace_seconds, to potentially
 # prevent data resurrection on tables with deletes. By default, this will run against all keyspaces and tables
 # except the ones specified on excluded_keyspaces and excluded_tables.
@@ -2129,4 +2123,4 @@ drop_compact_storage_enabled: false
 # - Do a rolling restart with all nodes starting with NONE. This sheds the extra cost of checking nodes versions and ensures 
 #   a stable cluster. If a node from a previous version was started by accident we won't any longer toggle behaviors as when UPGRADING.
 #
-storage_compatibility_mode: CASSANDRA_4
+storage_compatibility_mode: NONE

conf/harry-example.yaml

@@ -0,0 +1,95 @@
+#   Licensed to the Apache Software Foundation (ASF) under one
+#   or more contributor license agreements.  See the NOTICE file
+#   distributed with this work for additional information
+#   regarding copyright ownership.  The ASF licenses this file
+#   to you under the Apache License, Version 2.0 (the
+#   "License"); you may not use this file except in compliance
+#   with the License.  You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+seed: 1596731732524
+
+# Default schema provider generates random schema
+schema_provider:
+  fixed:
+    keyspace: harry
+    table: test_table
+    partition_keys:
+      pk1: bigint
+      pk2: ascii
+    clustering_keys:
+      ck1: ascii
+      ck2: bigint
+    regular_columns:
+      v1: ascii
+      v2: bigint
+      v3: ascii
+      v4: bigint
+    static_keys:
+      s1: ascii
+      s2: bigint
+      s3: ascii
+      s4: bigint
+
+# Clock is a component responsible for mapping _logical_ timestamps to _real-time_ ones.
+#
+# When reproducing test failures, and for validation purposes, a snapshot of such clock can
+# be taken to map a real-time timestamp from the value retrieved from the database in order
+# to map it back to the logical timestamp of the operation that wrote this value.
+clock:
+  offset:
+    offset: 1000
+
+drop_schema: false
+create_schema: true
+truncate_table: true
+
+# Partition descriptor selector controls how partitions is selected based on the current logical
+# timestamp. Default implementation is a sliding window of partition descriptors that will visit
+# one partition after the other in the window `slide_after_repeats` times. After that will
+# retire one partition descriptor, and pick one instead of it.
+partition_descriptor_selector:
+  default:
+    window_size: 10
+    slide_after_repeats: 100
+
+# Clustering descriptor selector controls how clusterings are picked within the partition:
+# how many rows there can be in a partition, how many rows will be visited for a logical timestamp,
+# how many operations there will be in batch, what kind of operations there will and how often
+# each kind of operation is going to occur.
+clustering_descriptor_selector:
+  default:
+    modifications_per_lts:
+      type: "constant"
+      constant: 2
+    rows_per_modification:
+      type: "constant"
+      constant: 2
+    operation_kind_weights:
+      DELETE_RANGE: 0
+      DELETE_SLICE: 0
+      DELETE_ROW: 0
+      DELETE_COLUMN: 0
+      DELETE_PARTITION: 0
+      DELETE_COLUMN_WITH_STATICS: 0
+      INSERT_WITH_STATICS: 50
+      INSERT: 50
+      UPDATE_WITH_STATICS: 50
+      UPDATE: 50
+    column_mask_bitsets: null
+    max_partition_size: 1000
+
+metric_reporter:
+  no_op: {}
+
+data_tracker:
+  locking:
+    max_seen_lts: -1
+    max_complete_lts: -1

pylib/cqlshlib/cql3handling.py

@@ -34,8 +34,8 @@ class UnexpectedTableStructure(UserWarning):
 
 
 SYSTEM_KEYSPACES = ('system', 'system_schema', 'system_traces', 'system_auth', 'system_distributed', 'system_views',
-                    'system_virtual_schema')
+                    'system_virtual_schema', 'system_cluster_metadata')
-NONALTERBALE_KEYSPACES = ('system', 'system_schema', 'system_views', 'system_virtual_schema')
+NONALTERBALE_KEYSPACES = ('system', 'system_schema', 'system_views', 'system_virtual_schema', 'system_cluster_metadata')
 
 
 class Cql3ParsingRuleSet(CqlParsingRuleSet):

pylib/cqlshlib/test/test_cqlsh_completion.py

@@ -589,7 +589,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
 
     def test_complete_in_drop_type(self):
         self.trycompletions('DROP TYPE ', choices=['IF', 'system_views.',
-                                                    'tags', 'system_traces.', 'system_distributed.',
+                                                    'tags', 'system_traces.', 'system_distributed.', 'system_cluster_metadata.',
                                                     'phone_number', 'quote_udt', 'band_info_type', 'address', 'system.', 'system_schema.',
                                                     'system_auth.', 'system_virtual_schema.', self.cqlsh.keyspace + '.'
                                                     ])
@@ -897,7 +897,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
         self.trycompletions('US', immediate='E ')
         self.trycompletions('USE ', choices=[self.cqlsh.keyspace, 'system', 'system_auth',
                                            'system_distributed', 'system_schema', 'system_traces', 'system_views',
-                                           'system_virtual_schema' ])
+                                           'system_virtual_schema', 'system_cluster_metadata' ])
 
     def test_complete_in_create_index(self):
         self.trycompletions('CREATE I', immediate='NDEX ')
@@ -994,6 +994,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
                                                      'system_traces.', 'songs', 'system_views.',
                                                      'system_virtual_schema.',
                                                      'system_schema.', 'system_distributed.',
+                                                     'system_cluster_metadata.',
                                                      self.cqlsh.keyspace + '.'])
         self.trycompletions('ALTER TABLE IF EXISTS new_table ADD ', choices=['<new_column_name>', 'IF'])
         self.trycompletions('ALTER TABLE IF EXISTS new_table ADD IF NOT EXISTS ', choices=['<new_column_name>'])
@@ -1021,7 +1022,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
         self.trycompletions('ALTER TYPE ', choices=['IF', 'system_views.',
                                                     'tags', 'system_traces.', 'system_distributed.',
                                                     'phone_number', 'quote_udt', 'band_info_type', 'address', 'system.', 'system_schema.',
-                                                    'system_auth.', 'system_virtual_schema.', self.cqlsh.keyspace + '.'
+                                                    'system_auth.', 'system_virtual_schema.', 'system_cluster_metadata.', self.cqlsh.keyspace + '.'
                                                     ])
         self.trycompletions('ALTER TYPE IF EXISTS new_type ADD ', choices=['<new_field_name>', 'IF'])
         self.trycompletions('ALTER TYPE IF EXISTS new_type ADD IF NOT EXISTS ', choices=['<new_field_name>'])

src/java/org/apache/cassandra/auth/AuthKeyspace.java

@@ -41,7 +41,7 @@ public final class AuthKeyspace
     {
     }
 
-    private static final int DEFAULT_RF = CassandraRelevantProperties.SYSTEM_AUTH_DEFAULT_RF.getInt();
+    public static final int DEFAULT_RF = CassandraRelevantProperties.SYSTEM_AUTH_DEFAULT_RF.getInt();
 
     /**
      * Generation is used as a timestamp for automatic table creation on startup.
@@ -68,79 +68,85 @@ public final class AuthKeyspace
 
     public static final long SUPERUSER_SETUP_DELAY = SUPERUSER_SETUP_DELAY_MS.getLong();
 
+    public static String ROLES_CQL = "CREATE TABLE IF NOT EXISTS %s ("
+                                     + "role text,"
+                                     + "is_superuser boolean,"
+                                     + "can_login boolean,"
+                                     + "salted_hash text,"
+                                     + "member_of set<text>,"
+                                     + "PRIMARY KEY(role))";
     private static final TableMetadata Roles =
         parse(ROLES,
               "role definitions",
-              "CREATE TABLE %s ("
+              ROLES_CQL);
-              + "role text,"
+
-              + "is_superuser boolean,"
+    public static String IDENTITY_TO_ROLES_CQL = "CREATE TABLE IF NOT EXISTS %s ("
-              + "can_login boolean,"
+                                                 + "identity text," // opaque identity string for use by role authenticators
-              + "salted_hash text,"
+                                                 + "role text,"
-              + "member_of set<text>,"
+                                                 + "PRIMARY KEY(identity))";
-              + "PRIMARY KEY(role))");
 
     private static final TableMetadata IdentityToRoles =
         parse(IDENTITY_TO_ROLES,
               "mtls authorized identities lookup table",
-              "CREATE TABLE %s ("
+              IDENTITY_TO_ROLES_CQL
-              + "identity text," // opaque identity string for use by role authenticators
-              + "role text,"
-              + "PRIMARY KEY(identity))"
           );
 
+    public static String ROLE_MEMBERS_CQL = "CREATE TABLE IF NOT EXISTS %s ("
+                                            + "role text,"
+                                            + "member text,"
+                                            + "PRIMARY KEY(role, member))";
     private static final TableMetadata RoleMembers =
         parse(ROLE_MEMBERS,
               "role memberships lookup table",
-              "CREATE TABLE %s ("
+              ROLE_MEMBERS_CQL);
-              + "role text,"
-              + "member text,"
-              + "PRIMARY KEY(role, member))");
 
+    public static String ROLE_PERMISSIONS_CQL = "CREATE TABLE IF NOT EXISTS %s ("
+                                                + "role text,"
+                                                + "resource text,"
+                                                + "permissions set<text>,"
+                                                + "PRIMARY KEY(role, resource))";
     private static final TableMetadata RolePermissions =
         parse(ROLE_PERMISSIONS,
               "permissions granted to db roles",
-              "CREATE TABLE %s ("
+              ROLE_PERMISSIONS_CQL);
-              + "role text,"
-              + "resource text,"
-              + "permissions set<text>,"
-              + "PRIMARY KEY(role, resource))");
 
+    public static String RESOURCE_ROLE_INDEX_CQL = "CREATE TABLE IF NOT EXISTS %s ("
+                                               + "resource text,"
+                                               + "role text,"
+                                               + "PRIMARY KEY(resource, role))";
     private static final TableMetadata ResourceRoleIndex =
         parse(RESOURCE_ROLE_INDEX,
               "index of db roles with permissions granted on a resource",
-              "CREATE TABLE %s ("
+              RESOURCE_ROLE_INDEX_CQL);
-              + "resource text,"
-              + "role text,"
-              + "PRIMARY KEY(resource, role))");
 
+    public static String NETWORK_PERMISSIONS_CQL = "CREATE TABLE IF NOT EXISTS %s ("
+                                                   + "role text, "
+                                                   + "dcs frozen<set<text>>, "
+                                                   + "PRIMARY KEY(role))";
     private static final TableMetadata NetworkPermissions =
         parse(NETWORK_PERMISSIONS,
               "user network permissions",
-              "CREATE TABLE %s ("
+              NETWORK_PERMISSIONS_CQL);
-              + "role text, "
+
-              + "dcs frozen<set<text>>, "
+    public static String CIDR_PERMISSIONS_CQL = "CREATE TABLE %s ("
-              + "PRIMARY KEY(role))");
+                                                + "role text, "
+                                                + "cidr_groups frozen<set<text>>, "
+                                                + "PRIMARY KEY(role))";
 
-    public static final String CIDR_PERMISSIONS_TBL_ROLE_COL_NAME = "role";
-    public static final String CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME = "cidr_groups";
     private static final TableMetadata CIDRPermissions =
     parse(CIDR_PERMISSIONS,
           "user cidr permissions",
-          "CREATE TABLE %s ("
+          CIDR_PERMISSIONS_CQL
-          + CIDR_PERMISSIONS_TBL_ROLE_COL_NAME + " text, "
-          + CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME + " frozen<set<text>>, "
-          + "PRIMARY KEY(" + CIDR_PERMISSIONS_TBL_ROLE_COL_NAME + "))"
     );
 
-    public static final String CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME = "cidr_group";
+    public static String CIDR_GROUPS_CQL = "CREATE TABLE %s ("
-    public static final String CIDR_GROUPS_TBL_CIDRS_COL_NAME = "cidrs";
+                                           + "cidr_group text, "
+                                           + "cidrs frozen<set<tuple<inet, smallint>>>, "
+                                           + "PRIMARY KEY(cidr_group))";
     private static final TableMetadata CIDRGroups =
     parse(CIDR_GROUPS,
           "cidr groups to cidrs mapping",
-          "CREATE TABLE %s ("
+          CIDR_GROUPS_CQL
-          + CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME + " text, "
-          + CIDR_GROUPS_TBL_CIDRS_COL_NAME + " frozen<set<tuple<inet, smallint>>>, "
-          + "PRIMARY KEY(" + CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME + "))"
     );
 
     private static TableMetadata parse(String name, String description, String cql)
@@ -158,7 +164,6 @@ public final class AuthKeyspace
                                        KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())),
                                        Tables.of(Roles, RoleMembers, RolePermissions,
                                                  ResourceRoleIndex, NetworkPermissions,
-                                                 CIDRPermissions, CIDRGroups,
+                                                 CIDRPermissions, CIDRGroups, IdentityToRoles));
-                                                 IdentityToRoles));
     }
 }

src/java/org/apache/cassandra/auth/CIDRGroupsMappingLoader.java

@@ -58,7 +58,7 @@ public class CIDRGroupsMappingLoader
         UntypedResultSet rows = cidrGroupsMappingManager.getCidrGroupsTableEntries();
         for (UntypedResultSet.Row row : rows)
         {
-            String cidrGroupName = row.getString(AuthKeyspace.CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME);
+            String cidrGroupName = row.getString("cidr_group");
             Set<Pair<InetAddress, Short>> cidrs = cidrGroupsMappingManager.retrieveCidrsFromRow(row);
 
             for (Pair<InetAddress, Short> cidr : cidrs)

src/java/org/apache/cassandra/auth/CIDRGroupsMappingManager.java

@@ -70,18 +70,15 @@ public class CIDRGroupsMappingManager implements CIDRGroupsMappingManagerMBean
         if (!MBeanWrapper.instance.isRegistered(MBEAN_NAME))
             MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
 
-        String getCidrGroupsQuery = String.format("SELECT %s FROM %s.%s",
+        String getCidrGroupsQuery = String.format("SELECT cidr_group FROM %s.%s",
-                                                  AuthKeyspace.CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME,
                                                   SchemaConstants.AUTH_KEYSPACE_NAME,
                                                   AuthKeyspace.CIDR_GROUPS);
         getCidrGroupsStatement = (SelectStatement) QueryProcessor.getStatement(getCidrGroupsQuery,
                                                                                ClientState.forInternalCalls());
 
-        String getCidrsForCidrGroupQuery = String.format("SELECT %s FROM %s.%s where %s = ?",
+        String getCidrsForCidrGroupQuery = String.format("SELECT cidrs FROM %s.%s where cidr_group = ?",
-                                                         AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME,
                                                          SchemaConstants.AUTH_KEYSPACE_NAME,
-                                                         AuthKeyspace.CIDR_GROUPS,
+                                                         AuthKeyspace.CIDR_GROUPS);
-                                                         AuthKeyspace.CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME);
         getCidrsForCidrGroupStatement = (SelectStatement) QueryProcessor.getStatement(getCidrsForCidrGroupQuery,
                                                                                       ClientState.forInternalCalls());
     }
@@ -125,11 +122,11 @@ public class CIDRGroupsMappingManager implements CIDRGroupsMappingManagerMBean
 
         for (UntypedResultSet.Row row : result)
         {
-            if (!row.has(AuthKeyspace.CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME))
+            if (!row.has("cidr_group"))
                 throw new IllegalStateException("Invalid row " + row + " in table: " +
                                                 SchemaConstants.AUTH_KEYSPACE_NAME + '.' + AuthKeyspace.CIDR_GROUPS);
 
-            availableCidrGroups.add(row.getString(AuthKeyspace.CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME));
+            availableCidrGroups.add(row.getString("cidr_group"));
         }
 
         return availableCidrGroups;
@@ -137,14 +134,13 @@ public class CIDRGroupsMappingManager implements CIDRGroupsMappingManagerMBean
 
     public Set<Pair<InetAddress, Short>> retrieveCidrsFromRow(UntypedResultSet.Row row)
     {
-        if (!row.has(AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME))
+        if (!row.has("cidrs"))
-            throw new RuntimeException("Invalid row, doesn't have column " +
+            throw new RuntimeException("Invalid row, doesn't have column cidrs");
-                                       AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME);
 
         Set<Pair<InetAddress, Short>> cidrs = new HashSet<>();
         TupleType tupleType = new TupleType(Arrays.asList(InetAddressType.instance, ShortType.instance));
 
-        Set<ByteBuffer> cidrAsTuples = row.getFrozenSet(AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME, tupleType);
+        Set<ByteBuffer> cidrAsTuples = row.getFrozenSet("cidrs", tupleType);
         for (ByteBuffer cidrAsTuple : cidrAsTuples)
         {
             ByteBuffer[] splits = tupleType.split(ByteBufferAccessor.instance, cidrAsTuple);
@@ -182,12 +178,10 @@ public class CIDRGroupsMappingManager implements CIDRGroupsMappingManagerMBean
             validCidrs.add(CIDR.getInstance(cidr));
         }
 
-        String query = String.format("UPDATE %s.%s SET %s = %s WHERE %s = '%s'",
+        String query = String.format("UPDATE %s.%s SET cidrs = %s WHERE cidr_group = '%s'",
                                      SchemaConstants.AUTH_KEYSPACE_NAME,
                                      AuthKeyspace.CIDR_GROUPS,
-                                     AuthKeyspace.CIDR_GROUPS_TBL_CIDRS_COL_NAME,
                                      getCidrTuplesSetString(validCidrs),
-                                     AuthKeyspace.CIDR_GROUPS_TBL_CIDR_GROUP_COL_NAME,
                                      cidrGroupName);
 
         process(query, CassandraAuthorizer.authWriteConsistencyLevel());

src/java/org/apache/cassandra/auth/CIDRPermissionsManager.java

@@ -64,11 +64,9 @@ public class CIDRPermissionsManager implements CIDRPermissionsManagerMBean, Auth
         if (!MBeanWrapper.instance.isRegistered(MBEAN_NAME))
             MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
 
-        String getCidrPermissionsOfUserQuery = String.format("SELECT %s FROM %s.%s WHERE %s = ?",
+        String getCidrPermissionsOfUserQuery = String.format("SELECT cidr_groups FROM %s.%s WHERE role = ?",
-                                                             AuthKeyspace.CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME,
                                                              SchemaConstants.AUTH_KEYSPACE_NAME,
-                                                             AuthKeyspace.CIDR_PERMISSIONS,
+                                                             AuthKeyspace.CIDR_PERMISSIONS);
-                                                             AuthKeyspace.CIDR_PERMISSIONS_TBL_ROLE_COL_NAME);
         getCidrPermissionsOfUserStatement = (SelectStatement) QueryProcessor.getStatement(getCidrPermissionsOfUserQuery,
                                                                                           ClientState.forInternalCalls());
     }
@@ -92,9 +90,9 @@ public class CIDRPermissionsManager implements CIDRPermissionsManagerMBean, Auth
 
         ResultMessage.Rows rows = select(getCidrPermissionsOfUserStatement, options);
         UntypedResultSet result = UntypedResultSet.create(rows.result);
-        if (!result.isEmpty() && result.one().has(AuthKeyspace.CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME))
+        if (!result.isEmpty() && result.one().has("cidr_groups"))
         {
-            return result.one().getFrozenSet(AuthKeyspace.CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME, UTF8Type.instance);
+            return result.one().getFrozenSet("cidr_groups", UTF8Type.instance);
         }
 
         return Collections.emptySet();
@@ -145,12 +143,10 @@ public class CIDRPermissionsManager implements CIDRPermissionsManagerMBean, Auth
      */
     public void setCidrGroupsForRole(RoleResource role, CIDRPermissions cidrPermissions)
     {
-        String query = String.format("UPDATE %s.%s SET %s = %s WHERE %s = '%s'",
+        String query = String.format("UPDATE %s.%s SET cidr_groups = %s WHERE role = '%s'",
                                      SchemaConstants.AUTH_KEYSPACE_NAME,
                                      AuthKeyspace.CIDR_PERMISSIONS,
-                                     AuthKeyspace.CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME,
                                      getCidrPermissionsSetString(cidrPermissions),
-                                     AuthKeyspace.CIDR_PERMISSIONS_TBL_ROLE_COL_NAME,
                                      role.getRoleName());
 
         process(query, CassandraAuthorizer.authWriteConsistencyLevel());
@@ -191,18 +187,16 @@ public class CIDRPermissionsManager implements CIDRPermissionsManagerMBean, Auth
 
             logger.info("Pre-warming CIDR permissions cache from cidr_permissions table");
             Map<RoleResource, CIDRPermissions> entries = new HashMap<>();
-            UntypedResultSet rows = process(String.format("SELECT %s, %s FROM %s.%s",
+            UntypedResultSet rows = process(String.format("SELECT role, cidr_groups FROM %s.%s",
-                                                          AuthKeyspace.CIDR_PERMISSIONS_TBL_ROLE_COL_NAME,
-                                                          AuthKeyspace.CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME,
                                                           SchemaConstants.AUTH_KEYSPACE_NAME,
                                                           AuthKeyspace.CIDR_PERMISSIONS),
                                             CassandraAuthorizer.authReadConsistencyLevel());
 
             for (UntypedResultSet.Row row : rows)
             {
-                RoleResource role = RoleResource.role(row.getString(AuthKeyspace.CIDR_PERMISSIONS_TBL_ROLE_COL_NAME));
+                RoleResource role = RoleResource.role(row.getString("role"));
                 CIDRPermissions.Builder builder = new CIDRPermissions.Builder();
-                Set<String> cidrGroups = row.getFrozenSet(AuthKeyspace.CIDR_PERMISSIONS_TBL_CIDR_GROUPS_COL_NAME,
+                Set<String> cidrGroups = row.getFrozenSet("cidr_groups",
                                                           UTF8Type.instance);
                 for (String cidrGroup : cidrGroups)
                     builder.add(cidrGroup);

src/java/org/apache/cassandra/auth/CassandraRoleManager.java

@@ -36,18 +36,19 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.FBUtilities;
 import org.mindrot.jbcrypt.BCrypt;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.AUTH_BCRYPT_GENSALT_LOG2_ROUNDS;
@@ -144,14 +145,21 @@ public class CassandraRoleManager implements IRoleManager
     }
 
     @Override
-    public void setup()
+    public void setup(boolean asyncRoleSetup)
     {
         loadRoleStatement();
         loadIdentityStatement();
-        scheduleSetupTask(() -> {
+        if (asyncRoleSetup)
+        {
+            scheduleSetupTask(() -> {
+                setupDefaultRole();
+                return null;
+            });
+        }
+        else
+        {
             setupDefaultRole();
-            return null;
+        }
-        });
     }
 
     @Override
@@ -417,7 +425,7 @@ public class CassandraRoleManager implements IRoleManager
      */
     private static void setupDefaultRole()
     {
-        if (StorageService.instance.getTokenMetadata().sortedTokens().isEmpty())
+        if (ClusterMetadata.current().tokenMap.tokens().isEmpty())
             throw new IllegalStateException("CassandraRoleManager skipped default role setup: no known tokens in ring");
 
         try
@@ -461,7 +469,7 @@ public class CassandraRoleManager implements IRoleManager
     {
         // The delay is to give the node a chance to see its peers before attempting the operation
         ScheduledExecutors.optionalTasks.scheduleSelfRecurring(() -> {
-            if (!StorageProxy.isSafeToPerformRead())
+            if (!StorageProxy.hasJoined())
             {
                 logger.trace("Setup task may not run due to it not being safe to perform reads... rescheduling");
                 scheduleSetupTask(setupTask);
@@ -487,7 +495,7 @@ public class CassandraRoleManager implements IRoleManager
         }
         catch (RequestValidationException e)
         {
-            throw new AssertionError(e); // not supposed to happen
+            throw new AssertionError(e + " " + FBUtilities.getJustLocalAddress()); // not supposed to happen
         }
     }
 

src/java/org/apache/cassandra/auth/DCPermissions.java

@@ -28,6 +28,7 @@ import com.google.common.collect.Sets;
 
 import org.apache.cassandra.dht.Datacenters;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.tcm.ClusterMetadata;
 
 public abstract class DCPermissions
 {
@@ -93,7 +94,7 @@ public abstract class DCPermissions
 
         public void validate()
         {
-            Set<String> unknownDcs = Sets.difference(subset, Datacenters.getValidDatacenters());
+            Set<String> unknownDcs = Sets.difference(subset, Datacenters.getValidDatacenters(ClusterMetadata.current()));
             if (!unknownDcs.isEmpty())
             {
                 throw new InvalidRequestException(String.format("Invalid value(s) for DATACENTERS '%s'," +

src/java/org/apache/cassandra/auth/IRoleManager.java

@@ -22,6 +22,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestValidationException;
@@ -226,7 +228,16 @@ public interface IRoleManager extends AuthCache.BulkLoader<RoleResource, Set<Rol
      *
      * For example, use this method to create any required keyspaces/column families.
      */
-    void setup();
+    default void setup()
+    {
+        setup(true);
+    }
+
+    /**
+     * Like the method above, but allows to disable async role setup, making it synchronous.
+     */
+    @VisibleForTesting
+    void setup(boolean asyncRoleSetup);
 
     /**
      * Each valid identity is associated with a role in the identity_to_role table, this method returns role

src/java/org/apache/cassandra/batchlog/BatchlogManager.java

@@ -36,6 +36,8 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.RateLimiter;
 import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.slf4j.Logger;
@@ -203,7 +205,7 @@ public class BatchlogManager implements BatchlogManagerMBean
 
         // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
         // max rate is scaled by the number of nodes in the cluster (same as for HHOM - see CASSANDRA-5272).
-        int endpointsCount = StorageService.instance.getTokenMetadata().getSizeOfAllEndpoints();
+        int endpointsCount = ClusterMetadata.current().directory.allJoinedEndpoints().size();
         if (endpointsCount <= 0)
         {
             logger.trace("Replay cancelled as there are no peers in the ring.");
@@ -234,7 +236,7 @@ public class BatchlogManager implements BatchlogManagerMBean
      */
     public void setRate(final int throttleInKB)
     {
-        int endpointsCount = StorageService.instance.getTokenMetadata().getSizeOfAllEndpoints();
+        int endpointsCount = ClusterMetadata.current().directory.allAddresses().size();
         if (endpointsCount > 0)
         {
             int endpointThrottleInKiB = throttleInKB / endpointsCount;
@@ -471,29 +473,27 @@ public class BatchlogManager implements BatchlogManagerMBean
                                                                                      Set<UUID> hintedNodes)
         {
             String ks = mutation.getKeyspaceName();
-            Keyspace keyspace = Keyspace.open(ks);
             Token tk = mutation.key().getToken();
+            ClusterMetadata metadata = ClusterMetadata.current();
+            KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaceMetadata(ks);
 
             // TODO: this logic could do with revisiting at some point, as it is unclear what its rationale is
             // we perform a local write, ignoring errors and inline in this thread (potentially slowing replay down)
             // effectively bumping CL for locally owned writes and also potentially stalling log replay if an error occurs
             // once we decide how it should work, it can also probably be simplified, and avoid constructing a ReplicaPlan directly
-            ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWriteLiveAndDown(keyspace, tk);
+            ReplicaLayout.ForTokenWrite allReplias = ReplicaLayout.forTokenWriteLiveAndDown(metadata, keyspaceMetadata, tk);
-            Replicas.temporaryAssertFull(liveAndDown.all()); // TODO in CASSANDRA-14549
+            ReplicaPlan.ForWrite replicaPlan = forReplayMutation(metadata, Keyspace.open(ks), tk);
 
-            Replica selfReplica = liveAndDown.all().selfIfPresent();
+            Replica selfReplica = allReplias.all().selfIfPresent();
             if (selfReplica != null)
                 mutation.apply();
 
-            ReplicaLayout.ForTokenWrite liveRemoteOnly = liveAndDown.filter(
+            for (Replica replica : allReplias.all())
-                    r -> FailureDetector.isReplicaAlive.test(r) && r != selfReplica);
-
-            for (Replica replica : liveAndDown.all())
             {
-                if (replica == selfReplica || liveRemoteOnly.all().contains(replica))
+                if (replica == selfReplica || replicaPlan.liveAndDown().contains(replica))
                     continue;
 
-                UUID hostId = StorageService.instance.getHostIdForEndpoint(replica.endpoint());
+                UUID hostId = metadata.directory.peerId(replica.endpoint()).toUUID();
                 if (null != hostId)
                 {
                     HintsService.instance.write(hostId, Hint.create(mutation, writtenAt));
@@ -501,15 +501,26 @@ public class BatchlogManager implements BatchlogManagerMBean
                 }
             }
 
-            ReplicaPlan.ForWrite replicaPlan = new ReplicaPlan.ForWrite(keyspace, liveAndDown.replicationStrategy(),
-                                                                        ConsistencyLevel.ONE, liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all());
             ReplayWriteResponseHandler<Mutation> handler = new ReplayWriteResponseHandler<>(replicaPlan, mutation, nanoTime());
             Message<Mutation> message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE);
-            for (Replica replica : liveRemoteOnly.all())
+            for (Replica replica : replicaPlan.liveAndDown())
                 MessagingService.instance().sendWriteWithCallback(message, replica, handler);
             return handler;
         }
 
+        public static ReplicaPlan.ForWrite forReplayMutation(ClusterMetadata metadata, Keyspace keyspace, Token token)
+        {
+            ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWriteLiveAndDown(metadata, keyspace.getMetadata(), token);
+            Replicas.temporaryAssertFull(liveAndDown.all()); // TODO in CASSANDRA-14549
+
+            Replica selfReplica = liveAndDown.all().selfIfPresent();
+            ReplicaLayout.ForTokenWrite liveRemoteOnly = liveAndDown.filter(r -> FailureDetector.isReplicaAlive.test(r) && r != selfReplica);
+
+            return new ReplicaPlan.ForWrite(keyspace, liveAndDown.replicationStrategy(),
+                                            ConsistencyLevel.ONE, liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all(),
+                                            (cm) -> forReplayMutation(cm, keyspace, token),
+                                            metadata.epoch);
+        }
         private static int gcgs(Collection<Mutation> mutations)
         {
             int gcgs = Integer.MAX_VALUE;

src/java/org/apache/cassandra/cache/AutoSavingCache.java

@@ -60,6 +60,7 @@ import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -219,12 +220,13 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
                 in = streamFactory.getInputStream(dataPath, crcPath);
 
                 //Check the schema has not changed since CFs are looked up by name which is ambiguous
-                UUID schemaVersion = new UUID(in.readLong(), in.readLong());
+                UUID expected = new UUID(in.readLong(), in.readLong());
-                if (!schemaVersion.equals(Schema.instance.getVersion()))
+                UUID actual = ClusterMetadata.current().schema.getVersion();
+                if (!expected.equals(actual))
                     throw new RuntimeException("Cache schema version "
-                                               + schemaVersion
+                                               + expected
                                                + " does not match current schema version "
-                                               + Schema.instance.getVersion());
+                                               + actual);
 
                 ArrayDeque<Future<Pair<K, V>>> futures = new ArrayDeque<>();
                 long loadByNanos = start + TimeUnit.SECONDS.toNanos(DatabaseDescriptor.getCacheLoadTimeout());

src/java/org/apache/cassandra/concurrent/NamedThreadFactory.java

@@ -17,12 +17,14 @@
  */
 package org.apache.cassandra.concurrent;
 
+import java.util.Arrays;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import io.netty.util.concurrent.FastThreadLocalThread;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
@@ -33,6 +35,8 @@ import org.apache.cassandra.utils.JVMStabilityInspector;
 
 public class NamedThreadFactory implements ThreadFactory
 {
+    public static final Boolean PRESERVE_THREAD_CREATION_STACKTRACE = CassandraRelevantProperties.TEST_PRESERVE_THREAD_CREATION_STACKTRACE.getBoolean();
+
     private static final AtomicInteger anonymousCounter = new AtomicInteger();
     private static volatile String globalPrefix;
 
@@ -159,11 +163,43 @@ public class NamedThreadFactory implements ThreadFactory
     public static Thread createThread(ThreadGroup threadGroup, Runnable runnable, String name, boolean daemon)
     {
         String prefix = globalPrefix;
-        Thread thread = new FastThreadLocalThread(threadGroup, runnable, prefix != null ? prefix + name : name);
+        Thread thread;
+        String threadName = prefix != null ? prefix + name : name;
+        if (PRESERVE_THREAD_CREATION_STACKTRACE)
+            thread = new InspectableFastThreadLocalThread(threadGroup, runnable, threadName);
+        else
+            thread = new FastThreadLocalThread(threadGroup, runnable, threadName);
         thread.setDaemon(daemon);
         return thread;
     }
 
+    public static class InspectableFastThreadLocalThread extends FastThreadLocalThread
+    {
+        public StackTraceElement[] creationTrace;
+
+        private void setStack()
+        {
+            creationTrace = Thread.currentThread().getStackTrace();
+            creationTrace = Arrays.copyOfRange(creationTrace, 2, creationTrace.length);
+        }
+
+        public InspectableFastThreadLocalThread() { super(); setStack(); }
+
+        public InspectableFastThreadLocalThread(Runnable target) { super(target); setStack(); }
+
+        public InspectableFastThreadLocalThread(ThreadGroup group, Runnable target) { super(group, target); setStack(); }
+
+        public InspectableFastThreadLocalThread(String name) { super(name); setStack(); }
+
+        public InspectableFastThreadLocalThread(ThreadGroup group, String name) { super(group, name); setStack(); }
+
+        public InspectableFastThreadLocalThread(Runnable target, String name) { super(target, name); setStack(); }
+
+        public InspectableFastThreadLocalThread(ThreadGroup group, Runnable target, String name) { super(group, target, name); setStack(); }
+
+        public InspectableFastThreadLocalThread(ThreadGroup group, Runnable target, String name, long stackSize) { super(group, target, name, stackSize); setStack(); }
+
+    }
     public static  <T extends Thread> T setupThread(T thread, int priority, ClassLoader contextClassLoader, Thread.UncaughtExceptionHandler uncaughtExceptionHandler)
     {
         thread.setPriority(priority);

src/java/org/apache/cassandra/concurrent/Stage.java

@@ -55,6 +55,8 @@ public enum Stage
     INTERNAL_RESPONSE (false, "InternalResponseStage", "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
     IMMEDIATE         (false, "ImmediateStage",        "internal", () -> 0,                                         null,                                            Stage::immediateExecutor),
     PAXOS_REPAIR      (false, "PaxosRepairStage",      "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
+    INTERNAL_METADATA (false, "InternalMetadataStage", "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
+    FETCH_LOG         (false, "MetadataFetchLogStage", "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage)
     ;
 
     public final String jmxName;

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -353,7 +353,7 @@ public enum CassandraRelevantProperties
     MEMTABLE_OVERHEAD_SIZE("cassandra.memtable.row_overhead_size", "-1"),
     MEMTABLE_SHARD_COUNT("cassandra.memtable.shard.count"),
     MEMTABLE_TRIE_SIZE_LIMIT("cassandra.trie_size_limit_mb"),
-    MIGRATION_DELAY("cassandra.migration_delay_ms", "60000"),
+    METRICS_REPORTER_CONFIG_FILE("cassandra.metricsReporterConfigFile"),
     /** Defines the maximum number of unique timed out queries that will be reported in the logs. Use a negative number to remove any limit. */
     MONITORING_MAX_OPERATIONS("cassandra.monitoring_max_operations", "50"),
     /** Defines the interval for reporting any operations that have timed out. */
@@ -468,7 +468,6 @@ public enum CassandraRelevantProperties
      */
     SAI_VECTOR_SEARCH_ORDER_CHUNK_SIZE("cassandra.sai.vector_search.order_chunk_size", "100000"),
 
-    SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
     SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
     SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
 
@@ -482,6 +481,7 @@ public enum CassandraRelevantProperties
     SET_SEP_THREAD_NAME("cassandra.set_sep_thread_name", "true"),
     SHUTDOWN_ANNOUNCE_DELAY_IN_MS("cassandra.shutdown_announce_in_ms", "2000"),
     SIZE_RECORDER_INTERVAL("cassandra.size_recorder_interval", "300"),
+    SKIP_GC_INSPECTOR("cassandra.skip_gc_inspector", "false"),
     SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE("cassandra.skip_paxos_repair_on_topology_change"),
     /** If necessary for operational purposes, permit certain keyspaces to be ignored for paxos topology repairs. */
     SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES("cassandra.skip_paxos_repair_on_topology_change_keyspaces"),
@@ -518,6 +518,30 @@ public enum CassandraRelevantProperties
     SYSTEM_AUTH_DEFAULT_RF("cassandra.system_auth.default_rf", "1"),
     SYSTEM_DISTRIBUTED_DEFAULT_RF("cassandra.system_distributed.default_rf", "3"),
     SYSTEM_TRACES_DEFAULT_RF("cassandra.system_traces.default_rf", "2"),
+
+    // transactional cluster metadata relevant properties
+    // TODO: not a fan of being forced to prefix these to satisfy the alphabetic ordering constraint
+    //       but it makes sense to group logically related properties together
+
+    TCM_ALLOW_TRANSFORMATIONS_DURING_UPGRADES("cassandra.allow_transformations_during_upgrades", "false"),
+    /**
+     * for obtaining acknowlegement from peers to make progress in multi-step operations
+     */
+    TCM_PROGRESS_BARRIER_BACKOFF_MILLIS("cassandra.progress_barrier_backoff_ms", "1000"),
+    TCM_PROGRESS_BARRIER_TIMEOUT_MILLIS("cassandra.progress_barrier_timeout_ms", "3600000"),
+    /**
+     * size of in-memory index of max epoch -> sealed period
+     */
+    TCM_RECENTLY_SEALED_PERIOD_INDEX_SIZE("cassandra.recently_sealed_period_index_size", "10"),
+
+    /**
+     * should replica groups in data placements be sorted to ensure the primary replica is first in the list
+     */
+    TCM_SORT_REPLICA_GROUPS("cassandra.sorted_replica_groups_enabled", "true"),
+    TCM_UNSAFE_BOOT_WITH_CLUSTERMETADATA("cassandra.unsafe_boot_with_clustermetadata", null),
+    TCM_USE_ATOMIC_LONG_PROCESSOR("cassandra.test.use_atomic_long_processor", "false"),
+    TCM_USE_NO_OP_REPLICATOR("cassandra.test.use_no_op_replicator", "false"),
+
     TEST_BBFAILHELPER_ENABLED("test.bbfailhelper.enabled"),
     TEST_BLOB_SHARED_SEED("cassandra.test.blob.shared.seed"),
     TEST_BYTEMAN_TRANSFORMATIONS_DEBUG("cassandra.test.byteman.transformations.debug"),
@@ -543,13 +567,16 @@ public enum CassandraRelevantProperties
     TEST_IGNORE_SIGAR("cassandra.test.ignore_sigar"),
     TEST_INVALID_LEGACY_SSTABLE_ROOT("invalid-legacy-sstable-root"),
     TEST_JVM_DTEST_DISABLE_SSL("cassandra.test.disable_ssl"),
+    TEST_JVM_SHUTDOWN_MESSAGING_GRACEFULLY("cassandra.test.messagingService.gracefulShutdown", "false"),
     TEST_LEGACY_SSTABLE_ROOT("legacy-sstable-root"),
     TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT("org.caffinitas.ohc.segmentCount"),
+    TEST_PRESERVE_THREAD_CREATION_STACKTRACE("cassandra.test.preserve_thread_creation_stacktrace", "false"),
     TEST_RANDOM_SEED("cassandra.test.random.seed"),
     TEST_READ_ITERATION_DELAY_MS("cassandra.test.read_iteration_delay_ms", "0"),
     TEST_REUSE_PREPARED("cassandra.test.reuse_prepared", "true"),
     TEST_ROW_CACHE_SIZE("cassandra.test.row_cache_size"),
     TEST_SERIALIZATION_WRITES("cassandra.test-serialization-writes"),
+    TEST_SIGAR_NATIVE_LOGGING("sigar.nativeLogging", "true"),
     TEST_SIMULATOR_DEBUG("cassandra.test.simulator.debug"),
     TEST_SIMULATOR_DETERMINISM_CHECK("cassandra.test.simulator.determinismcheck", "none"),
     TEST_SIMULATOR_LIVENESS_CHECK("cassandra.test.simulator.livenesscheck", "true"),
@@ -590,7 +617,9 @@ public enum CassandraRelevantProperties
     USE_NIX_RECURSIVE_DELETE("cassandra.use_nix_recursive_delete"),
     /** Gossiper compute expiration timeout. Default value 3 days. */
     VERY_LONG_TIME_MS("cassandra.very_long_time_ms", "259200000"),
-    WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS("cassandra.wait_for_tracing_events_timeout_secs", "0");
+    WAIT_FOR_TRACING_EVENTS_TIMEOUT_SECS("cassandra.wait_for_tracing_events_timeout_secs", "0"),
+    ;
+
 
     static
     {

src/java/org/apache/cassandra/config/Config.java

@@ -171,6 +171,15 @@ public class Config
 
     public volatile DurationSpec.LongMillisecondsBound stream_transfer_task_timeout = new DurationSpec.LongMillisecondsBound("12h");
 
+    public volatile DurationSpec.LongMillisecondsBound cms_await_timeout = new DurationSpec.LongMillisecondsBound("120000ms");
+    public volatile int cms_default_max_retries = 10;
+    public volatile DurationSpec.IntMillisecondsBound cms_default_retry_backoff = new DurationSpec.IntMillisecondsBound("50ms");
+    /**
+     * How often we should snapshot the cluster metadata.
+     */
+    public volatile int metadata_snapshot_frequency = 100;
+
+
     public volatile double phi_convict_threshold = 8.0;
 
     public int concurrent_reads = 32;
@@ -1243,5 +1252,21 @@ public class Config
 
     public double severity_during_decommission = 0;
 
-    public StorageCompatibilityMode storage_compatibility_mode = StorageCompatibilityMode.CASSANDRA_4;
+    // TODO Revisit MessagingService::current_version
+    public StorageCompatibilityMode storage_compatibility_mode = StorageCompatibilityMode.NONE;
+
+    /**
+     * For the purposes of progress barrier we only support ALL, EACH_QUORUM, QUORUM, LOCAL_QUORUM, ANY, and ONE.
+     *
+     * We will still try all consistency levels above the lowest acceptable, and only fall back to it if we can not
+     * collect enough nodes.
+     */
+    public volatile ConsistencyLevel progress_barrier_min_consistency_level = ConsistencyLevel.EACH_QUORUM;
+    public volatile boolean log_out_of_token_range_requests = true;
+    public volatile boolean reject_out_of_token_range_requests = true;
+    public volatile ConsistencyLevel progress_barrier_default_consistency_level = ConsistencyLevel.EACH_QUORUM;
+
+    public volatile DurationSpec.LongMillisecondsBound progress_barrier_timeout = new DurationSpec.LongMillisecondsBound("3600000ms");
+    public volatile DurationSpec.LongMillisecondsBound progress_barrier_backoff = new DurationSpec.LongMillisecondsBound("1000ms");
+    public boolean unsafe_tcm_mode = false;
 }

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -29,10 +29,12 @@ import java.net.UnknownHostException;
 import java.nio.file.FileStore;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.Enumeration;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -84,6 +86,7 @@ import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.fql.FullQueryLoggerOptions;
 import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.big.BigFormat;
@@ -104,6 +107,7 @@ import org.apache.cassandra.security.EncryptionContext;
 import org.apache.cassandra.security.JREProvider;
 import org.apache.cassandra.security.SSLFactory;
 import org.apache.cassandra.service.CacheService.CacheType;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.paxos.Paxos;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.StorageCompatibilityMode;
@@ -135,6 +139,7 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.UNSAFE_SYS
 import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.BYTES_PER_SECOND;
 import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.MEBIBYTES_PER_SECOND;
 import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.MEBIBYTES;
+import static org.apache.cassandra.db.ConsistencyLevel.*;
 import static org.apache.cassandra.io.util.FileUtils.ONE_GIB;
 import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
 import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome;
@@ -592,12 +597,12 @@ public class DatabaseDescriptor
         if (conf.repair_session_space.toMebibytes() < 1)
             throw new ConfigurationException("repair_session_space must be > 0, but was " + conf.repair_session_space);
         else if (conf.repair_session_space.toMebibytes() > (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576)))
-            logger.warn("A repair_session_space of " + conf.repair_session_space+ " mebibytes is likely to cause heap pressure");
+            logger.warn("A repair_session_space of " + conf.repair_session_space + " mebibytes is likely to cause heap pressure");
 
         checkForLowestAcceptedTimeouts(conf);
 
         long valueInBytes = conf.native_transport_max_frame_size.toBytes();
-        if (valueInBytes < 0 || valueInBytes > Integer.MAX_VALUE-1)
+        if (valueInBytes < 0 || valueInBytes > Integer.MAX_VALUE - 1)
         {
             throw new ConfigurationException(String.format("native_transport_max_frame_size must be positive value < %dB, but was %dB",
                                                            Integer.MAX_VALUE,
@@ -770,8 +775,8 @@ public class DatabaseDescriptor
         {
             // if prepared_statements_cache_size option was set to "auto" then size of the cache should be "max(1/256 of Heap (in MiB), 10MiB)"
             preparedStatementsCacheSizeInMiB = (conf.prepared_statements_cache_size == null)
-                                              ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256))
+                                               ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256))
-                                              : conf.prepared_statements_cache_size.toMebibytes();
+                                               : conf.prepared_statements_cache_size.toMebibytes();
 
             if (preparedStatementsCacheSizeInMiB == 0)
                 throw new NumberFormatException(); // to escape duplicating error message
@@ -789,8 +794,8 @@ public class DatabaseDescriptor
         {
             // if key_cache_size option was set to "auto" then size of the cache should be "min(5% of Heap (in MiB), 100MiB)
             keyCacheSizeInMiB = (conf.key_cache_size == null)
-                               ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024)), 100)
+                                ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024)), 100)
-                               : conf.key_cache_size.toMebibytes();
+                                : conf.key_cache_size.toMebibytes();
 
             if (keyCacheSizeInMiB < 0)
                 throw new NumberFormatException(); // to escape duplicating error message
@@ -808,8 +813,8 @@ public class DatabaseDescriptor
         {
             // if counter_cache_size option was set to "auto" then size of the cache should be "min(2.5% of Heap (in MiB), 50MiB)
             counterCacheSizeInMiB = (conf.counter_cache_size == null)
-                                   ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.025 / 1024 / 1024)), 50)
+                                    ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.025 / 1024 / 1024)), 50)
-                                   : conf.counter_cache_size.toMebibytes();
+                                    : conf.counter_cache_size.toMebibytes();
 
             if (counterCacheSizeInMiB < 0)
                 throw new NumberFormatException(); // to escape duplicating error message
@@ -817,15 +822,15 @@ public class DatabaseDescriptor
         catch (NumberFormatException e)
         {
             throw new ConfigurationException("counter_cache_size option was set incorrectly to '"
-                                             + (conf.counter_cache_size !=null ?conf.counter_cache_size.toString() : null) + "', supported values are <integer> >= 0.", false);
+                                             + (conf.counter_cache_size != null ? conf.counter_cache_size.toString() : null) + "', supported values are <integer> >= 0.", false);
         }
 
         try
         {
             // if paxosCacheSizeInMiB option was set to "auto" then size of the cache should be "min(1% of Heap (in MB), 50MB)
             paxosCacheSizeInMiB = (conf.paxos_cache_size == null)
-                    ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.01 / 1024 / 1024)), 50)
+                                  ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.01 / 1024 / 1024)), 50)
-                    : conf.paxos_cache_size.toMebibytes();
+                                  : conf.paxos_cache_size.toMebibytes();
 
             if (paxosCacheSizeInMiB < 0)
                 throw new NumberFormatException(); // to escape duplicating error message
@@ -833,7 +838,7 @@ public class DatabaseDescriptor
         catch (NumberFormatException e)
         {
             throw new ConfigurationException("paxos_cache_size option was set incorrectly to '"
-                    + conf.paxos_cache_size + "', supported values are <integer> >= 0.", false);
+                                             + conf.paxos_cache_size + "', supported values are <integer> >= 0.", false);
         }
 
         // we need this assignment for the Settings virtual table - CASSANDRA-17735
@@ -841,8 +846,8 @@ public class DatabaseDescriptor
 
         // if set to empty/"auto" then use 5% of Heap size
         indexSummaryCapacityInMiB = (conf.index_summary_capacity == null)
-                                   ? Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024))
+                                    ? Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024))
-                                   : conf.index_summary_capacity.toMebibytes();
+                                    : conf.index_summary_capacity.toMebibytes();
 
         if (indexSummaryCapacityInMiB < 0)
             throw new ConfigurationException("index_summary_capacity option was set incorrectly to '"
@@ -860,7 +865,7 @@ public class DatabaseDescriptor
         if (conf.allow_extra_insecure_udfs)
             logger.warn("Allowing java.lang.System.* access in UDFs is dangerous and not recommended. Set allow_extra_insecure_udfs: false to disable.");
 
-        if(conf.scripted_user_defined_functions_enabled)
+        if (conf.scripted_user_defined_functions_enabled)
             throw new ConfigurationException("JavaScript user-defined functions were removed in CASSANDRA-18252. " +
                                              "Hooks are planned to be introduced as part of CASSANDRA-17280");
 
@@ -896,7 +901,7 @@ public class DatabaseDescriptor
             throw new ConfigurationException("max_value_size must be positive", false);
         else if (conf.max_value_size.toMebibytes() >= 2048)
             throw new ConfigurationException("max_value_size must be smaller than 2048, but was "
-                    + conf.max_value_size.toString(), false);
+                                             + conf.max_value_size.toString(), false);
 
         switch (conf.disk_optimization_strategy)
         {
@@ -968,6 +973,20 @@ public class DatabaseDescriptor
             throw new ConfigurationException(String.format("Invalid configuration. Heap dump is enabled but cannot create heap dump output path: %s.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
 
         conf.sai_options.validate();
+
+        List<ConsistencyLevel> progressBarrierCLsArr = Arrays.asList(ALL, EACH_QUORUM, LOCAL_QUORUM, QUORUM, ONE, NODE_LOCAL);
+        Set<ConsistencyLevel> progressBarrierCls = new HashSet<>(progressBarrierCLsArr);
+        if (!progressBarrierCls.contains(conf.progress_barrier_min_consistency_level))
+        {
+            throw new ConfigurationException(String.format("Invalid value for progress_barrier_min_consistency_level %s. Allowed values: %s",
+                                                           conf.progress_barrier_min_consistency_level, progressBarrierCLsArr));
+        }
+
+        if (!progressBarrierCls.contains(conf.progress_barrier_default_consistency_level))
+        {
+            throw new ConfigurationException(String.format("Invalid value for.progress_barrier_default_consistency_level %s. Allowed values: %s",
+                                                           conf.progress_barrier_default_consistency_level, progressBarrierCLsArr));
+        }
     }
 
     @VisibleForTesting
@@ -1264,7 +1283,7 @@ public class DatabaseDescriptor
         try
         {
             Class<?> seedProviderClass = Class.forName(conf.seed_provider.class_name);
-            seedProvider = (SeedProvider)seedProviderClass.getConstructor(Map.class).newInstance(conf.seed_provider.parameters);
+            seedProvider = (SeedProvider) seedProviderClass.getConstructor(Map.class).newInstance(conf.seed_provider.parameters);
         }
         // there are about 5 checked exceptions that could be thrown here.
         catch (Exception e)
@@ -1278,42 +1297,42 @@ public class DatabaseDescriptor
     @VisibleForTesting
     static void checkForLowestAcceptedTimeouts(Config conf)
     {
-        if(conf.read_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.read_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("read_request_timeout", conf.read_request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.read_request_timeout = new DurationSpec.LongMillisecondsBound("10ms");
         }
 
-        if(conf.range_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.range_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("range_request_timeout", conf.range_request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.range_request_timeout = new DurationSpec.LongMillisecondsBound("10ms");
         }
 
-        if(conf.request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("request_timeout", conf.request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.request_timeout = new DurationSpec.LongMillisecondsBound("10ms");
         }
 
-        if(conf.write_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.write_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("write_request_timeout", conf.write_request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.write_request_timeout = new DurationSpec.LongMillisecondsBound("10ms");
         }
 
-        if(conf.cas_contention_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.cas_contention_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("cas_contention_timeout", conf.cas_contention_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.cas_contention_timeout = new DurationSpec.LongMillisecondsBound("10ms");
         }
 
-        if(conf.counter_write_request_timeout.toMilliseconds()< LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.counter_write_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("counter_write_request_timeout", conf.counter_write_request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.counter_write_request_timeout = new DurationSpec.LongMillisecondsBound("10ms");
         }
-        if(conf.truncate_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
+        if (conf.truncate_request_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
         {
             logInfo("truncate_request_timeout", conf.truncate_request_timeout, LOWEST_ACCEPTED_TIMEOUT);
             conf.truncate_request_timeout = LOWEST_ACCEPTED_TIMEOUT;
@@ -1518,7 +1537,9 @@ public class DatabaseDescriptor
 
     private static long tryGetSpace(String dir, PathUtils.IOToLongFunction<FileStore> getSpace)
     {
-        return PathUtils.tryGetSpace(new File(dir).toPath(), getSpace, e -> { throw new ConfigurationException("Unable check disk space in '" + dir + "'. Perhaps the Cassandra user does not have the necessary permissions"); });
+        return PathUtils.tryGetSpace(new File(dir).toPath(), getSpace, e -> {
+            throw new ConfigurationException("Unable check disk space in '" + dir + "'. Perhaps the Cassandra user does not have the necessary permissions");
+        });
     }
 
     public static IEndpointSnitch createEndpointSnitch(boolean dynamic, String snitchClassName) throws ConfigurationException
@@ -1546,6 +1567,7 @@ public class DatabaseDescriptor
     {
         DatabaseDescriptor.cryptoProvider = cryptoProvider;
     }
+
     public static IAuthenticator getAuthenticator()
     {
         return authenticator;
@@ -1618,7 +1640,7 @@ public class DatabaseDescriptor
     {
         int defaultCidrGroupsCacheRefreshInterval = 5; // mins
 
-        if (conf.cidr_authorizer == null  || conf.cidr_authorizer.parameters == null)
+        if (conf.cidr_authorizer == null || conf.cidr_authorizer.parameters == null)
             return defaultCidrGroupsCacheRefreshInterval;
 
         String cidrGroupsCacheRefreshInterval = conf.cidr_authorizer.parameters.get("cidr_groups_cache_refresh_interval");
@@ -1632,7 +1654,7 @@ public class DatabaseDescriptor
     {
         int defaultIpCacheMaxSize = 100;
 
-        if (conf.cidr_authorizer == null  || conf.cidr_authorizer.parameters == null)
+        if (conf.cidr_authorizer == null || conf.cidr_authorizer.parameters == null)
             return defaultIpCacheMaxSize;
 
         String ipCacheMaxSize = conf.cidr_authorizer.parameters.get("ip_cache_max_size");
@@ -1675,8 +1697,8 @@ public class DatabaseDescriptor
     public static int getPermissionsUpdateInterval()
     {
         return conf.permissions_update_interval == null
-             ? conf.permissions_validity.toMilliseconds()
+               ? conf.permissions_validity.toMilliseconds()
-             : conf.permissions_update_interval.toMilliseconds();
+               : conf.permissions_update_interval.toMilliseconds();
     }
 
     public static void setPermissionsUpdateInterval(int updateInterval)
@@ -1720,8 +1742,8 @@ public class DatabaseDescriptor
     public static int getRolesUpdateInterval()
     {
         return conf.roles_update_interval == null
-             ? conf.roles_validity.toMilliseconds()
+               ? conf.roles_validity.toMilliseconds()
-             : conf.roles_update_interval.toMilliseconds();
+               : conf.roles_update_interval.toMilliseconds();
     }
 
     public static void setRolesCacheActiveUpdate(boolean update)
@@ -1852,7 +1874,7 @@ public class DatabaseDescriptor
         }
         catch (ConfigurationException e)
         {
-            throw new IllegalArgumentException("Bad configuration; unable to start server: "+e.getMessage());
+            throw new IllegalArgumentException("Bad configuration; unable to start server: " + e.getMessage());
         }
         catch (FSWriteError e)
         {
@@ -1872,6 +1894,13 @@ public class DatabaseDescriptor
 
     /* For tests ONLY, don't use otherwise or all hell will break loose. Tests should restore value at the end. */
     public static IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner)
+    {
+        IPartitioner old = setOnlyPartitionerUnsafe(newPartitioner);
+        StorageService.instance.valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
+        return old;
+    }
+
+    public static IPartitioner setOnlyPartitionerUnsafe(IPartitioner newPartitioner)
     {
         IPartitioner old = partitioner;
         partitioner = newPartitioner;
@@ -1882,6 +1911,7 @@ public class DatabaseDescriptor
     {
         return snitch;
     }
+
     public static void setEndpointSnitch(IEndpointSnitch eps)
     {
         snitch = eps;
@@ -1909,7 +1939,7 @@ public class DatabaseDescriptor
 
     public static void setColumnIndexSizeInKiB(int val)
     {
-        conf.column_index_size = val != -1 ? createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val,"column_index_size") : null;
+        conf.column_index_size = val != -1 ? createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val, "column_index_size") : null;
     }
 
     public static int getColumnIndexCacheSize()
@@ -1924,7 +1954,7 @@ public class DatabaseDescriptor
 
     public static void setColumnIndexCacheSize(int val)
     {
-        conf.column_index_cache_size = createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val,"column_index_cache_size");
+        conf.column_index_cache_size = createIntKibibyteBoundAndEnsureItIsValidForByteConversion(val, "column_index_cache_size");
     }
 
     public static int getBatchSizeWarnThreshold()
@@ -1954,7 +1984,7 @@ public class DatabaseDescriptor
 
     public static void setBatchSizeWarnThresholdInKiB(int threshold)
     {
-        conf.batch_size_warn_threshold = createIntKibibyteBoundAndEnsureItIsValidForByteConversion(threshold,"batch_size_warn_threshold");
+        conf.batch_size_warn_threshold = createIntKibibyteBoundAndEnsureItIsValidForByteConversion(threshold, "batch_size_warn_threshold");
     }
 
     public static void setBatchSizeFailThresholdInKiB(int threshold)
@@ -2021,7 +2051,8 @@ public class DatabaseDescriptor
         try
         {
             return UUID.fromString(REPLACE_NODE.getString());
-        } catch (NullPointerException e)
+        }
+        catch (NullPointerException e)
         {
             return null;
         }
@@ -2378,7 +2409,7 @@ public class DatabaseDescriptor
     public static void setStreamThroughputOutboundMebibytesPerSecAsInt(int value)
     {
         if (MEBIBYTES_PER_SECOND.toMegabitsPerSecond(value) >= Integer.MAX_VALUE)
-            throw new IllegalArgumentException("stream_throughput_outbound: " + value  +
+            throw new IllegalArgumentException("stream_throughput_outbound: " + value +
                                                " is too large; it should be less than " +
                                                Integer.MAX_VALUE + " in megabits/s");
 
@@ -2502,10 +2533,10 @@ public class DatabaseDescriptor
     public static String[] getLocalSystemKeyspacesDataFileLocations()
     {
         if (useSpecificLocationForLocalSystemData())
-            return new String[] {conf.local_system_data_file_directory};
+            return new String[]{ conf.local_system_data_file_directory };
 
-        return conf.data_file_directories.length == 0  ? conf.data_file_directories
+        return conf.data_file_directories.length == 0 ? conf.data_file_directories
-                                                       : new String[] {conf.data_file_directories[0]};
+                                                      : new String[]{ conf.data_file_directories[0] };
     }
 
     /**
@@ -2562,11 +2593,11 @@ public class DatabaseDescriptor
         conf.flush_compression = compression;
     }
 
-   /**
+    /**
-    * Maximum number of buffers in the compression pool. The default value is 3, it should not be set lower than that
+     * Maximum number of buffers in the compression pool. The default value is 3, it should not be set lower than that
-    * (one segment in compression, one written to, one in reserve); delays in compression may cause the log to use
+     * (one segment in compression, one written to, one in reserve); delays in compression may cause the log to use
-    * more, depending on how soon the sync policy stops all writing threads.
+     * more, depending on how soon the sync policy stops all writing threads.
-    */
+     */
     public static int getCommitLogMaxCompressionBuffersInPool()
     {
         return conf.commitlog_max_compression_buffers_in_pool;
@@ -2634,6 +2665,7 @@ public class DatabaseDescriptor
      * Update commitlog_segment_size in the tests.
      * {@link CommitLogSegmentManagerCDC} uses the CommitLogSegmentSize to estimate the file size on allocation.
      * It is important to keep the value unchanged for the estimation to be correct.
+     *
      * @param sizeMebibytes
      */
     @VisibleForTesting /* Only for testing */
@@ -2729,7 +2761,7 @@ public class DatabaseDescriptor
      * refer to it as native address although some places still call it RPC address. It's not thrift RPC anymore
      * so native is more appropriate. The address alone is not enough to uniquely identify this instance because
      * multiple instances might use the same interface with different ports.
-     *
+     * <p>
      * May be null, please use {@link FBUtilities#getBroadcastNativeAddressAndPort()} instead.
      */
     public static InetAddress getBroadcastRpcAddress()
@@ -2829,8 +2861,8 @@ public class DatabaseDescriptor
     }
 
     /**
-     *  This is the port used with RPC address for the native protocol to communicate with clients. Now that thrift RPC
+     * This is the port used with RPC address for the native protocol to communicate with clients. Now that thrift RPC
-     *  is no longer in use there is no RPC port.
+     * is no longer in use there is no RPC port.
      */
     public static int getNativeTransportPort()
     {
@@ -2872,7 +2904,7 @@ public class DatabaseDescriptor
     /**
      * If this value is set to <= 0 it will move auth requests to the standard request pool regardless of the current
      * size of the {@link org.apache.cassandra.transport.Dispatcher#authExecutor}'s active size.
-     *
+     * <p>
      * see {@link org.apache.cassandra.transport.Dispatcher#dispatch} for executor selection
      */
     public static void setNativeTransportMaxAuthThreads(int threads)
@@ -3122,7 +3154,7 @@ public class DatabaseDescriptor
     {
         DurationSpec.IntMillisecondsBound blockMillis = conf.periodic_commitlog_sync_lag_block;
         return blockMillis == null
-               ? (long)(getCommitLogSyncPeriod() * 1.5)
+               ? (long) (getCommitLogSyncPeriod() * 1.5)
                : blockMillis.toMilliseconds();
     }
 
@@ -3211,6 +3243,7 @@ public class DatabaseDescriptor
     {
         conf.auto_snapshot = autoSnapshot;
     }
+
     @VisibleForTesting
     public static boolean getAutoSnapshot()
     {
@@ -3314,7 +3347,7 @@ public class DatabaseDescriptor
     public static File getSerializedCachePath(CacheType cacheType, String version, String extension)
     {
         String name = cacheType.toString()
-                + (version == null ? "" : '-' + version + '.' + extension);
+                      + (version == null ? "" : '-' + version + '.' + extension);
         return new File(conf.saved_caches_directory, name);
     }
 
@@ -3322,6 +3355,7 @@ public class DatabaseDescriptor
     {
         return conf.dynamic_snitch_update_interval.toMilliseconds();
     }
+
     public static void setDynamicUpdateInterval(int dynamicUpdateInterval)
     {
         conf.dynamic_snitch_update_interval = new DurationSpec.IntMillisecondsBound(dynamicUpdateInterval);
@@ -3331,6 +3365,7 @@ public class DatabaseDescriptor
     {
         return conf.dynamic_snitch_reset_interval.toMilliseconds();
     }
+
     public static void setDynamicResetInterval(int dynamicResetInterval)
     {
         conf.dynamic_snitch_reset_interval = new DurationSpec.IntMillisecondsBound(dynamicResetInterval);
@@ -3399,7 +3434,7 @@ public class DatabaseDescriptor
 
     public static long getMaxHintsFileSize()
     {
-        return  conf.max_hints_file_size.toBytesInLong();
+        return conf.max_hints_file_size.toBytesInLong();
     }
 
     public static ParameterizedClass getHintsCompression()
@@ -3513,7 +3548,9 @@ public class DatabaseDescriptor
         conf.key_cache_migrate_during_compaction = migrateCacheEntry;
     }
 
-    /** This method can return negative number for disabled */
+    /**
+     * This method can return negative number for disabled
+     */
     public static int getSSTablePreemptiveOpenIntervalInMiB()
     {
         if (conf.sstable_preemptive_open_interval == null)
@@ -3521,7 +3558,9 @@ public class DatabaseDescriptor
         return conf.sstable_preemptive_open_interval.toMebibytes();
     }
 
-    /** Negative number for disabled */
+    /**
+     * Negative number for disabled
+     */
     public static void setSSTablePreemptiveOpenIntervalInMiB(int mib)
     {
         if (mib < 0)
@@ -3809,8 +3848,10 @@ public class DatabaseDescriptor
         {
             switch (datamodel)
             {
-                case "64": return true;
+                case "64":
-                case "32": return false;
+                    return true;
+                case "32":
+                    return false;
             }
         }
         String arch = OS_ARCH.getString();
@@ -4073,7 +4114,7 @@ public class DatabaseDescriptor
 
     public static FullQueryLoggerOptions getFullQueryLogOptions()
     {
-        return  conf.full_query_logging_options;
+        return conf.full_query_logging_options;
     }
 
     public static void setFullQueryLogOptions(FullQueryLoggerOptions options)
@@ -4572,7 +4613,10 @@ public class DatabaseDescriptor
         conf.row_index_read_size_fail_threshold = value;
     }
 
-    public static int getDefaultKeyspaceRF() { return conf.default_keyspace_rf; }
+    public static int getDefaultKeyspaceRF()
+    {
+        return conf.default_keyspace_rf;
+    }
 
     public static void setDefaultKeyspaceRF(int value) throws IllegalArgumentException
     {
@@ -4665,11 +4709,13 @@ public class DatabaseDescriptor
         }
     }
 
-    public static DurationSpec.IntSecondsBound getStreamingSlowEventsLogTimeout() {
+    public static DurationSpec.IntSecondsBound getStreamingSlowEventsLogTimeout()
+    {
         return conf.streaming_slow_events_log_timeout;
     }
 
-    public static void setStreamingSlowEventsLogTimeout(String value) {
+    public static void setStreamingSlowEventsLogTimeout(String value)
+    {
         DurationSpec.IntSecondsBound next = new DurationSpec.IntSecondsBound(value);
         if (!conf.streaming_slow_events_log_timeout.equals(next))
         {
@@ -4785,6 +4831,7 @@ public class DatabaseDescriptor
      * both the more evolved cassandra.yaml approach but also the -XX param to override it on a one-off basis so you don't
      * have to change the full config of a node or a cluster in order to get a heap dump from a single node that's
      * misbehaving.
+     *
      * @return the absolute path of the -XX param if provided, else the heap_dump_path in cassandra.yaml
      */
     public static Path getHeapDumpPath()
@@ -4911,4 +4958,94 @@ public class DatabaseDescriptor
     {
         return conf == null ? new RepairRetrySpec() : conf.repair.retries;
     }
+
+    public static int getCmsDefaultRetryMaxTries()
+    {
+        return conf.cms_default_max_retries;
+    }
+
+    public static void setCmsDefaultRetryMaxTries(int value)
+    {
+        conf.cms_default_max_retries = value;
+    }
+
+    public static DurationSpec getDefaultRetryBackoff()
+    {
+        return conf.cms_default_retry_backoff;
+    }
+
+    public static DurationSpec getCmsAwaitTimeout()
+    {
+        return conf.cms_await_timeout;
+    }
+
+    public static int getMetadataSnapshotFrequency()
+    {
+        return conf.metadata_snapshot_frequency;
+    }
+
+    public static void setMetadataSnapshotFrequency(int frequency)
+    {
+        conf.metadata_snapshot_frequency = frequency;
+    }
+
+    public static ConsistencyLevel getProgressBarrierMinConsistencyLevel()
+    {
+        return conf.progress_barrier_min_consistency_level;
+    }
+
+    public static void setProgressBarrierMinConsistencyLevel(ConsistencyLevel newLevel)
+    {
+        conf.progress_barrier_min_consistency_level = newLevel;
+    }
+
+    public static boolean getLogOutOfTokenRangeRequests()
+    {
+        return conf.log_out_of_token_range_requests;
+    }
+
+    public static void setLogOutOfTokenRangeRequests(boolean enabled)
+    {
+        conf.log_out_of_token_range_requests = enabled;
+    }
+
+    public static boolean getRejectOutOfTokenRangeRequests()
+    {
+        return conf.reject_out_of_token_range_requests;
+    }
+
+    public static void setRejectOutOfTokenRangeRequests(boolean enabled)
+    {
+        conf.reject_out_of_token_range_requests = enabled;
+    }
+
+    public static ConsistencyLevel getProgressBarrierDefaultConsistencyLevel()
+    {
+        return conf.progress_barrier_default_consistency_level;
+    }
+
+    public static long getProgressBarrierTimeout(TimeUnit unit)
+    {
+        return conf.progress_barrier_timeout.to(unit);
+    }
+
+    public static long getProgressBarrierBackoff(TimeUnit unit)
+    {
+        return conf.progress_barrier_backoff.to(unit);
+    }
+
+    public static void setProgressBarrierTimeout(long timeOutInMillis)
+    {
+        conf.progress_barrier_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
+    }
+
+    public static void setProgressBarrierBackoff(long timeOutInMillis)
+    {
+        conf.progress_barrier_backoff = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
+    }
+
+    public static boolean getUnsafeTCMMode()
+    {
+        return conf.unsafe_tcm_mode;
+    }
 }

src/java/org/apache/cassandra/cql3/QueryProcessor.java

@@ -18,28 +18,63 @@
 package org.apache.cassandra.cql3;
 
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
 import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.antlr.runtime.*;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import org.antlr.runtime.RecognitionException;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.cql3.selection.ResultSetBuilder;
+import org.apache.cassandra.cql3.statements.BatchStatement;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.QualifiedStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.cql3.statements.schema.AlterSchemaStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.exceptions.CassandraException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.IsBootstrappingException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.CQLMetrics;
 import org.apache.cassandra.metrics.ClientRequestMetrics;
 import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
 import org.apache.cassandra.net.Message;
@@ -49,27 +84,20 @@ import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaChangeListener;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.cql3.functions.Function;
+import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.cql3.functions.FunctionName;
-import org.apache.cassandra.cql3.selection.ResultSetBuilder;
-import org.apache.cassandra.cql3.statements.*;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.partitions.PartitionIterators;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.exceptions.*;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.metrics.CQLMetrics;
-import org.apache.cassandra.service.*;
 import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MD5Digest;
+import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
 
@@ -206,7 +234,6 @@ public class QueryProcessor implements QueryHandler
 
     private QueryProcessor()
     {
-        Schema.instance.registerListener(new StatementInvalidatingListener());
     }
 
     @VisibleForTesting
@@ -273,7 +300,7 @@ public class QueryProcessor implements QueryHandler
 
     private ResultMessage processNodeLocalWrite(CQLStatement statement, QueryState queryState, QueryOptions options)
     {
-        ClientRequestMetrics  levelMetrics = ClientRequestsMetricsHolder.writeMetricsForLevel(ConsistencyLevel.NODE_LOCAL);
+        ClientRequestMetrics levelMetrics = ClientRequestsMetricsHolder.writeMetricsForLevel(ConsistencyLevel.NODE_LOCAL);
         ClientRequestMetrics globalMetrics = ClientRequestsMetricsHolder.writeMetrics;
 
         long startTime = nanoTime();
@@ -428,11 +455,16 @@ public class QueryProcessor implements QueryHandler
             qualifiedStatement.setKeyspace(clientState);
             keyspace = qualifiedStatement.keyspace();
         }
-
         // Note: if 2 threads prepare the same query, we'll live so don't bother synchronizing
         CQLStatement statement = raw.prepare(clientState);
         statement.validate(clientState);
 
+        // Set CQL string for AlterSchemaStatement as this is used to serialize the transformation
+        // in the cluster metadata log
+        if (statement instanceof AlterSchemaStatement)
+            ((AlterSchemaStatement)statement).setCql(query);
+
+
         if (isInternal)
             return new Prepared(statement, "", fullyQualified, keyspace);
         else
@@ -641,7 +673,7 @@ public class QueryProcessor implements QueryHandler
 
         synchronized (this)
         {
-            CassandraVersion minVersion = Gossiper.instance.getMinVersion(DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+            CassandraVersion minVersion = ClusterMetadata.current().directory.clusterMinVersion.cassandraVersion;
             if (minVersion != null && minVersion.compareTo(NEW_PREPARED_STATEMENT_BEHAVIOUR_SINCE_40, true) >= 0)
             {
                 logger.info("Fully upgraded to at least {}", minVersion);
@@ -854,7 +886,13 @@ public class QueryProcessor implements QueryHandler
             ((QualifiedStatement) statement).setKeyspace(clientState);
 
         Tracing.trace("Preparing statement");
-        return statement.prepare(clientState);
+        CQLStatement prepared = statement.prepare(clientState);
+        // Set CQL string for AlterSchemaStatement as this is used to serialize the transformation
+        // in the cluster metadata log
+        if (prepared instanceof AlterSchemaStatement)
+            ((AlterSchemaStatement) prepared).setCql(queryStr);
+
+        return prepared;
     }
 
     public static <T extends CQLStatement.Raw> T parseStatement(String queryStr, Class<T> klass, String type) throws SyntaxException
@@ -917,6 +955,11 @@ public class QueryProcessor implements QueryHandler
         preparedStatements.asMap().clear();
     }
 
+    public static void registerStatementInvalidatingListener()
+    {
+        Schema.instance.registerListener(new StatementInvalidatingListener());
+    }
+
     private static class StatementInvalidatingListener implements SchemaChangeListener
     {
         private static void removeInvalidPreparedStatements(String ksName, String cfName)
@@ -1019,7 +1062,7 @@ public class QueryProcessor implements QueryHandler
         {
             // in case there are other overloads, we have to remove all overloads since argument type
             // matching may change (due to type casting)
-            if (Schema.instance.getKeyspaceMetadata(ksName).userFunctions.get(new FunctionName(ksName, functionName)).size() > 1)
+            if (!Schema.instance.getKeyspaceMetadata(ksName).userFunctions.get(new FunctionName(ksName, functionName)).isEmpty())
                 removeInvalidPreparedStatementsForFunction(ksName, functionName);
         }
 

src/java/org/apache/cassandra/cql3/functions/UDAggregate.java

@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.cql3.functions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
 
@@ -30,13 +31,20 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.schema.CQLTypeParser;
 import org.apache.cassandra.schema.Difference;
 import org.apache.cassandra.schema.UserFunctions;
+import org.apache.cassandra.schema.Types;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.db.TypeSizes.sizeof;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
@@ -44,6 +52,8 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
  */
 public class UDAggregate extends UserFunction implements AggregateFunction
 {
+    public static final Serializer serializer = new Serializer();
+
     protected static final Logger logger = LoggerFactory.getLogger(UDAggregate.class);
 
     private final UDFDataType stateType;
@@ -375,4 +385,69 @@ public class UDAggregate extends UserFunction implements AggregateFunction
         return builder.append(";")
                       .toString();
     }
+
+    // Not quite a MetadataSerializer, or even a UDTAwareMetadataSerializer, as it needs the collection of UDFs during deserialization.
+    public static class Serializer
+    {
+        public void serialize(UDAggregate t, DataOutputPlus out, Version version) throws IOException
+        {
+            out.writeUTF(t.name().keyspace);
+            out.writeUTF(t.name().name);
+            out.writeInt(t.argumentsList().size());
+            for (String arg : t.argumentsList())
+                out.writeUTF(arg);
+            out.writeUTF(t.returnType().asCQL3Type().toString());
+            out.writeUTF(t.stateFunction.name().name);
+            out.writeUTF(t.stateType.toAbstractType().asCQL3Type().toString());
+            out.writeBoolean(t.finalFunction() != null);
+            if (t.finalFunction() != null)
+                out.writeUTF(t.finalFunction().name().name);
+            out.writeBoolean(t.initialCondition() != null);
+            if (t.initialCondition() != null)
+                ByteBufferUtil.writeWithShortLength(t.initialCondition(), out);
+        }
+
+        public UDAggregate deserialize(DataInputPlus in, Types types, Collection<UDFunction> functions, Version version) throws IOException
+        {
+            String ks = in.readUTF();
+            String name = in.readUTF();
+            FunctionName fn = new FunctionName(ks, name);
+            int argCount = in.readInt();
+            List<AbstractType<?>> argList = new ArrayList<>(argCount);
+            for (int i = 0; i < argCount; i++)
+                argList.add(CQLTypeParser.parse(ks, in.readUTF(), types).udfType());
+            AbstractType<?> returnType = CQLTypeParser.parse(ks, in.readUTF(), types).udfType();
+            FunctionName stateFunction = new FunctionName(ks, in.readUTF());
+            AbstractType<?> stateType = CQLTypeParser.parse(ks, in.readUTF(), types).udfType();
+            boolean hasFinalFunction = in.readBoolean();
+            FunctionName finalFunction = null;
+            if (hasFinalFunction)
+                finalFunction = new FunctionName(ks, in.readUTF());
+            boolean hasInitialCondition = in.readBoolean();
+            ByteBuffer initCond = null;
+            if (hasInitialCondition)
+                initCond = ByteBufferUtil.readWithShortLength(in);
+            return UDAggregate.create(functions, fn, argList, returnType, stateFunction, finalFunction, stateType, initCond);
+        }
+
+        public long serializedSize(UDAggregate t, Version version)
+        {
+            long size = sizeof(t.name().keyspace) + sizeof(t.name().name);
+
+            size += sizeof(t.argumentsList().size());
+            for (String arg : t.argumentsList())
+                size += sizeof(arg);
+
+            size += sizeof(t.returnType().asCQL3Type().toString());
+            size += sizeof(t.stateFunction.name().name);
+            size += sizeof(t.stateType.toAbstractType().asCQL3Type().toString());
+            size += sizeof(t.finalFunction() != null);
+            if (t.finalFunction() != null)
+                size += sizeof(t.finalFunction().name().name);
+            size += sizeof(t.initialCondition() != null);
+            if (t.initialCondition() != null)
+                size += ByteBufferUtil.serializedSizeWithShortLength(t.initialCondition());
+            return size;
+        }
+    }
 }

src/java/org/apache/cassandra/cql3/functions/UDFunction.java

@@ -17,11 +17,13 @@
  */
 package org.apache.cassandra.cql3.functions;
 
+import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadMXBean;
 import java.net.InetAddress;
 import java.net.URL;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashSet;
@@ -34,6 +36,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 
 import com.google.common.base.Objects;
 import com.google.common.collect.Lists;
@@ -50,6 +53,10 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.exceptions.FunctionExecutionException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.serialization.UDTAwareMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.tracing.Tracing;
@@ -59,6 +66,8 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.db.TypeSizes.*;
+import static org.apache.cassandra.schema.SchemaKeyspace.bbToString;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
@@ -66,6 +75,8 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
  */
 public abstract class UDFunction extends UserFunction implements ScalarFunction
 {
+    public static final Serializer serializer = new Serializer();
+
     protected static final Logger logger = LoggerFactory.getLogger(UDFunction.class);
 
     static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
@@ -732,4 +743,66 @@ public abstract class UDFunction extends UserFunction implements ScalarFunction
             return super.loadClass(name);
         }
     }
+
+    public static class Serializer implements UDTAwareMetadataSerializer<UDFunction>
+    {
+        public void serialize(UDFunction t, DataOutputPlus out, Version version) throws IOException
+        {
+            out.writeUTF(t.name().keyspace);
+            out.writeUTF(t.name().name);
+            out.writeUTF(t.body());
+            out.writeUTF(t.language());
+            out.writeUTF(t.returnType().asCQL3Type().toString());
+            out.writeBoolean(t.isCalledOnNullInput());
+            List<String> arguments = t.argNames().stream().map(c -> bbToString(c.bytes)).collect(Collectors.toList());
+            out.writeInt(arguments.size());
+            for (String argument : arguments)
+                out.writeUTF(argument);
+
+            out.writeInt(t.argumentsList().size());
+            for (String type : t.argumentsList())
+                out.writeUTF(type);
+        }
+
+        public UDFunction deserialize(DataInputPlus in, Types types, Version version) throws IOException
+        {
+            String keyspace = in.readUTF();
+            String name = in.readUTF();
+            FunctionName fn = new FunctionName(keyspace, name);
+            String body = in.readUTF();
+            String language = in.readUTF();
+            AbstractType<?> returnType = CQLTypeParser.parse(keyspace, in.readUTF(), types).udfType();
+            boolean isCalledOnNullInput = in.readBoolean();
+            int argumentCount = in.readInt();
+            List<ColumnIdentifier> arguments = new ArrayList<>(argumentCount);
+            for (int i = 0; i < argumentCount; i++)
+                arguments.add(new ColumnIdentifier(in.readUTF(), true));
+
+            int argumentTypeCount = in.readInt();
+            List<AbstractType<?>> argTypes = new ArrayList<>(argumentTypeCount);
+            for (int i = 0; i < argumentTypeCount; i++)
+                argTypes.add(CQLTypeParser.parse(keyspace, in.readUTF(), types).udfType());
+
+            return UDFunction.create(fn, arguments, argTypes, returnType, isCalledOnNullInput, language, body);
+        }
+
+        public long serializedSize(UDFunction t, Version version)
+        {
+            long size = sizeof(t.name().keyspace);
+            size += sizeof(t.name().name);
+            size += sizeof(t.body());
+            size += sizeof(t.language());
+            size += sizeof(t.returnType().asCQL3Type().toString());
+            size += sizeof(t.isCalledOnNullInput());
+            List<String> arguments = t.argNames().stream().map(c -> bbToString(c.bytes)).collect(Collectors.toList());
+            size += sizeof(arguments.size());
+            for (String argument : arguments)
+                size += sizeof(argument);
+
+            size += sizeof(t.argumentsList().size());
+            for (String type : t.argumentsList())
+                size += sizeof(type);
+            return size;
+        }
+    }
 }

src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java

@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.cql3.functions.masking;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -40,10 +41,19 @@ import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.cql3.functions.FunctionResolver;
 import org.apache.cassandra.cql3.functions.ScalarFunction;
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.CQLTypeParser;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
+import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.vint.VIntCoding;
 
 import static java.lang.String.format;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
@@ -67,6 +77,7 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.invalidReq
  */
 public class ColumnMask
 {
+    public static Serializer serializer = new Serializer();
     public static final String DISABLED_ERROR_MESSAGE = "Cannot mask columns because dynamic data masking is not " +
                                                         "enabled. You can enable it with the " +
                                                         "dynamic_data_masking_enabled property on cassandra.yaml";
@@ -268,4 +279,72 @@ public class ColumnMask
             return format("%s(%s)", name, StringUtils.join(rawPartialArguments, ", "));
         }
     }
+
+    public static class Serializer
+    {
+        public void serialize(ColumnMask columnMask, DataOutputPlus out, Version version) throws IOException
+        {
+            out.writeUTF(columnMask.function.name().keyspace);
+            out.writeUTF(columnMask.function.name().name);
+            List<AbstractType<?>> argTypes = columnMask.partialArgumentTypes();
+            int numArgs = argTypes.size();
+            out.writeUnsignedVInt32(numArgs);
+            for (int i = 0; i < numArgs; i++)
+            {
+                out.writeUTF(argTypes.get(i).asCQL3Type().toString());
+                ByteBuffer value = columnMask.partialArgumentValues[i];
+                out.writeBoolean(value != null);
+                if (value != null)
+                    ByteBufferUtil.writeWithVIntLength(value, out);
+            }
+        }
+
+        public ColumnMask deserialize(DataInputPlus in, String keyspace, AbstractType<?> columnType, Types types, UserFunctions functions, Version version) throws IOException
+        {
+            FunctionName functionName = new FunctionName(in.readUTF(), in.readUTF());
+
+            int numArgs = in.readUnsignedVInt32();
+            List<AbstractType<?>> argTypes = new ArrayList<>(numArgs + 1);
+            argTypes.set(0, columnType);
+            ByteBuffer[] partialArgValues = new ByteBuffer[numArgs];
+            for (int i = 0; i < numArgs; i++)
+            {
+                AbstractType<?> argType = CQLTypeParser.parse(keyspace, in.readUTF(), types);
+                argTypes.set(i + 1,  argType);
+                boolean valuePresent = in.readBoolean();
+                partialArgValues[i] = valuePresent ? null : ByteBufferUtil.readWithVIntLength(in);
+            }
+
+            Function function = FunctionResolver.get(keyspace, functionName, argTypes, null, null, null, functions);
+            if (function == null)
+            {
+                throw new AssertionError(format("Unable to find masking function %s(%s)", functionName, argTypes));
+            }
+            else if (!(function instanceof ScalarFunction))
+            {
+                throw new AssertionError(format("Function %s is not a scalar masking function", function));
+            }
+            return new ColumnMask((ScalarFunction) function, partialArgValues);
+        }
+
+        public long serializedSize(ColumnMask columnMask, Version version)
+        {
+            List<AbstractType<?>> argTypes = columnMask.partialArgumentTypes();
+            int numArgs = argTypes.size();
+            long size = TypeSizes.sizeof(columnMask.function.name().keyspace) +
+                        TypeSizes.sizeof(columnMask.function.name().name) +
+                        VIntCoding.computeUnsignedVIntSize(numArgs);
+
+            for (int i = 0; i < numArgs; i++)
+            {
+                size += TypeSizes.sizeof(argTypes.get(i).asCQL3Type().toString());
+                size += TypeSizes.BOOL_SIZE;
+                ByteBuffer value = columnMask.partialArgumentValues[i];
+                if (value != null)
+                    size += ByteBufferUtil.serializedSizeWithVIntLength(value);
+            }
+            return size;
+        }
+    }
+
 }

src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java

@@ -177,8 +177,9 @@ public final class StatementRestrictions
     {
         this(type, table, allowFiltering);
 
-        final IndexRegistry indexRegistry = type.allowUseOfSecondaryIndices() ? IndexRegistry.obtain(table) : null;
+        final IndexRegistry indexRegistry = type.allowUseOfSecondaryIndices() && allowUseOfSecondaryIndices
-
+                                            ? IndexRegistry.obtain(table)
+                                            : null;
         /*
          * WHERE clause. For a given entity, rules are:
          *   - EQ relation conflicts with anything else (including a 2nd EQ)

src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java

@@ -132,13 +132,9 @@ public abstract class DescribeStatement<T> extends CQLStatement.Raw implements C
     @Override
     public ResultMessage executeLocally(QueryState state, QueryOptions options)
     {
-        DistributedSchema schema = Schema.instance.getDistributedSchemaBlocking();
+        Keyspaces keyspaces = Schema.instance.distributedAndLocalKeyspaces();
-
+        UUID schemaVersion = Schema.instance.getVersion();
-        Keyspaces keyspaces = Keyspaces.builder()
+        keyspaces = keyspaces.with(VirtualKeyspaceRegistry.instance.virtualKeyspacesMetadata());
-                                       .add(schema.getKeyspaces())
-                                       .add(Schema.instance.getLocalKeyspaces())
-                                       .add(VirtualKeyspaceRegistry.instance.virtualKeyspacesMetadata())
-                                       .build();
 
         PagingState pagingState = options.getPagingState();
 
@@ -156,7 +152,7 @@ public abstract class DescribeStatement<T> extends CQLStatement.Raw implements C
         //   (vint bytes) serialized schema hash (currently the result of Keyspaces.hashCode())
         //
 
-        long offset = getOffset(pagingState, schema.getVersion());
+        long offset = getOffset(pagingState, schemaVersion);
         int pageSize = options.getPageSize();
 
         Stream<? extends T> stream = describe(state.getClientState(), keyspaces);
@@ -173,7 +169,7 @@ public abstract class DescribeStatement<T> extends CQLStatement.Raw implements C
         ResultSet result = new ResultSet(resultMetadata, rows);
 
         if (pageSize > 0 && rows.size() == pageSize)
-            result.metadata.setHasMorePages(getPagingState(offset + pageSize, schema.getVersion()));
+            result.metadata.setHasMorePages(getPagingState(offset + pageSize, schemaVersion));
 
         return new ResultMessage.Rows(result);
     }

src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java

@@ -82,7 +82,7 @@ public class PropertyDefinitions
         return properties.containsKey(name);
     }
 
-    protected String getString(String name) throws SyntaxException
+    public String getString(String name) throws SyntaxException
     {
         Object val = properties.get(name);
         if (val == null)

src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java

@@ -18,21 +18,19 @@
 package org.apache.cassandra.cql3.statements.schema;
 
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLStatement;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.LocalStrategy;
@@ -42,7 +40,11 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.utils.FBUtilities;
@@ -52,9 +54,10 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_UNSA
 
 public final class AlterKeyspaceStatement extends AlterSchemaStatement
 {
+    private static final Logger logger = LoggerFactory.getLogger(AlterKeyspaceStatement.class);
+
     private static final boolean allow_alter_rf_during_range_movement = ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT.getBoolean();
     private static final boolean allow_unsafe_transient_changes = ALLOW_UNSAFE_TRANSIENT_CHANGES.getBoolean();
-    private final HashSet<String> clientWarnings = new HashSet<>();
 
     private final KeyspaceAttributes attrs;
     private final boolean ifExists;
@@ -66,10 +69,11 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         attrs.validate();
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
         {
@@ -83,17 +87,29 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement
         if (attrs.getReplicationStrategyClass() != null && attrs.getReplicationStrategyClass().equals(SimpleStrategy.class.getSimpleName()))
             Guardrails.simpleStrategyEnabled.ensureEnabled(state);
 
+        if (keyspace.params.replication.isMeta() && !keyspace.name.equals(SchemaConstants.METADATA_KEYSPACE_NAME))
+            throw ire("Can not alter a keyspace to use MetaReplicationStrategy");
+
         if (newKeyspace.params.replication.klass.equals(LocalStrategy.class))
             throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
 
-        newKeyspace.params.validate(keyspaceName, state);
+        newKeyspace.params.validate(keyspaceName, state, metadata);
+        newKeyspace.replicationStrategy.validate(metadata);
 
         validateNoRangeMovements();
-        validateTransientReplication(keyspace.createReplicationStrategy(), newKeyspace.createReplicationStrategy());
+        validateTransientReplication(keyspace, newKeyspace);
 
-        Keyspaces res = schema.withAddedOrUpdated(newKeyspace);
+        // Because we used to not properly validate unrecognized options, we only log a warning if we find one.
+        try
+        {
+            newKeyspace.replicationStrategy.validateExpectedOptions(metadata);
+        }
+        catch (ConfigurationException e)
+        {
+            logger.warn("Ignoring {}", e.getMessage());
+        }
 
-        return res;
+        return schema.withAddedOrUpdated(newKeyspace);
     }
 
     SchemaChange schemaChangeEvent(KeyspacesDiff diff)
@@ -109,13 +125,14 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement
     @Override
     Set<String> clientWarnings(KeyspacesDiff diff)
     {
+        HashSet<String> clientWarnings = new HashSet<>();
         if (diff.isEmpty())
             return clientWarnings;
 
         KeyspaceDiff keyspaceDiff = diff.altered.get(0);
 
-        AbstractReplicationStrategy before = keyspaceDiff.before.createReplicationStrategy();
+        AbstractReplicationStrategy before = keyspaceDiff.before.replicationStrategy;
-        AbstractReplicationStrategy after = keyspaceDiff.after.createReplicationStrategy();
+        AbstractReplicationStrategy after = keyspaceDiff.after.replicationStrategy;
 
         if (before.getReplicationFactor().fullReplicas < after.getReplicationFactor().fullReplicas)
             clientWarnings.add("When increasing replication factor you need to run a full (-full) repair to distribute the data.");
@@ -128,29 +145,36 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement
         if (allow_alter_rf_during_range_movement)
             return;
 
-        Stream<InetAddressAndPort> unreachableNotAdministrativelyInactive =
+        ClusterMetadata metadata = ClusterMetadata.current();
-            Gossiper.instance.getUnreachableMembers().stream().filter(endpoint -> !FBUtilities.getBroadcastAddressAndPort().equals(endpoint) &&
+        NodeId nodeId = metadata.directory.peerId(FBUtilities.getBroadcastAddressAndPort());
-                                                                      !Gossiper.instance.isAdministrativelyInactiveState(endpoint));
+        Set<InetAddressAndPort> notNormalEndpoints = metadata.directory.states.entrySet().stream().filter(e -> !e.getKey().equals(nodeId)).filter(e -> {
-        Stream<InetAddressAndPort> endpoints = Stream.concat(Gossiper.instance.getLiveMembers().stream(),
+            switch (e.getValue())
-                                                             unreachableNotAdministrativelyInactive);
+            {
-        List<InetAddressAndPort> notNormalEndpoints = endpoints.filter(endpoint -> !FBUtilities.getBroadcastAddressAndPort().equals(endpoint) &&
+                case BOOTSTRAPPING:
-                                                                                   !Gossiper.instance.getEndpointStateForEndpoint(endpoint).isNormalState())
+                case LEAVING:
-                                                               .collect(Collectors.toList());
+                case MOVING:
+                    return true;
+                default:
+                    return false;
+            }
+
+        }).map(e -> metadata.directory.endpoint(e.getKey())).collect(Collectors.toSet());
+
         if (!notNormalEndpoints.isEmpty())
         {
             throw new ConfigurationException("Cannot alter RF while some endpoints are not in normal state (no range movements): " + notNormalEndpoints);
         }
     }
 
-    private void validateTransientReplication(AbstractReplicationStrategy oldStrategy, AbstractReplicationStrategy newStrategy)
+    private void validateTransientReplication(KeyspaceMetadata current, KeyspaceMetadata proposed)
     {
         //If there is no read traffic there are some extra alterations you can safely make, but this is so atypical
         //that a good default is to not allow unsafe changes
         if (allow_unsafe_transient_changes)
             return;
 
-        ReplicationFactor oldRF = oldStrategy.getReplicationFactor();
+        ReplicationFactor oldRF = current.replicationStrategy.getReplicationFactor();
-        ReplicationFactor newRF = newStrategy.getReplicationFactor();
+        ReplicationFactor newRF = proposed.replicationStrategy.getReplicationFactor();
 
         int oldTrans = oldRF.transientReplicas();
         int oldFull = oldRF.fullReplicas;
@@ -162,19 +186,13 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement
             if (DatabaseDescriptor.getNumTokens() > 1)
                 throw new ConfigurationException(String.format("Transient replication is not supported with vnodes yet"));
 
-            Keyspace ks = Keyspace.open(keyspaceName);
-            for (ColumnFamilyStore cfs : ks.getColumnFamilyStores())
-            {
-                if (cfs.viewManager.hasViews())
-                {
-                    throw new ConfigurationException("Cannot use transient replication on keyspaces using materialized views");
-                }
 
-                if (cfs.indexManager.hasIndexes())
+            if (!current.views.isEmpty())
-                {
+                throw new ConfigurationException("Cannot use transient replication on keyspaces using materialized views");
+
+            for (TableMetadata table : current.tables)
+                if (!table.indexes.isEmpty())
                     throw new ConfigurationException("Cannot use transient replication on keyspaces using secondary indexes");
-                }
-            }
         }
 
         //This is true right now because the transition from transient -> full lacks the pending state

src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
@@ -41,12 +42,46 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
 {
     protected final String keyspaceName; // name of the keyspace affected by the statement
     protected ClientState state;
+    // TODO: not sure if this is going to stay the same, or will be replaced by more efficient serialization/sanitation means
+    // or just `toString` for every statement
+    private String cql;
 
     protected AlterSchemaStatement(String keyspaceName)
     {
         this.keyspaceName = keyspaceName;
     }
 
+    public void setCql(String cql)
+    {
+        this.cql = cql;
+    }
+
+    @Override
+    public String cql()
+    {
+        assert cql != null;
+        return cql;
+    }
+
+    @Override
+    public void enterExecution()
+    {
+        ClientWarn.instance.pauseCapture();
+        ClientState localState = state;
+        if (localState != null)
+            localState.pauseGuardrails();
+    }
+
+    @Override
+    public void exitExecution()
+    {
+        ClientWarn.instance.resumeCapture();
+        ClientState localState = state;
+        if (localState != null)
+            localState.resumeGuardrails();
+    }
+
+    // TODO: validation should be performed during application
     public void validate(ClientState state)
     {
         // validation is performed while executing the statement, in apply()
@@ -57,7 +92,7 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
 
     public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
     {
-        return execute(state, false);
+        return execute(state);
     }
 
     @Override
@@ -68,7 +103,7 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
 
     public ResultMessage executeLocally(QueryState state, QueryOptions options)
     {
-        return execute(state, true);
+        return execute(state);
     }
 
     /**
@@ -100,7 +135,7 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
         return ImmutableSet.of();
     }
 
-    public ResultMessage execute(QueryState state, boolean locally)
+    public ResultMessage execute(QueryState state)
     {
         if (SchemaConstants.isLocalSystemKeyspace(keyspaceName))
             throw ire("System keyspace '%s' is not user-modifiable", keyspaceName);
@@ -110,12 +145,24 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
             throw ire("Virtual keyspace '%s' is not user-modifiable", keyspaceName);
 
         validateKeyspaceName();
+        // Perform a 'dry-run' attempt to apply the transformation locally before submitting to the CMS. This can save a
+        // round trip to the CMS for things syntax errors, but also fail fast for things like configuration errors.
+        // Such failures may be dependent on the specific node's config (for things like guardrails/memtable
+        // config/etc), but executing a schema change which has already been committed by the CMS should always succeed
+        // or else the node cannot make progress on any subsequent metadata changes. For this reason, validation errors
+        // during execution are trapped and the node will fall back to safe default config wherever possible. Attempting
+        // to apply the SchemaTransformation at this point will catch any such error which occurs locally before
+        // submission to the CMS, but it can't guarantee that the statement can be applied as-is on every node in the
+        // cluster, as config can be heterogenous falling back to safe defaults may occur on some nodes.
+        ClusterMetadata metadata = ClusterMetadata.current();
+        apply(metadata);
 
-        SchemaTransformationResult result = Schema.instance.transform(this, locally);
+        ClusterMetadata result = Schema.instance.submit(this);
 
-        clientWarnings(result.diff).forEach(ClientWarn.instance::warn);
+        KeyspacesDiff diff = Keyspaces.diff(metadata.schema.getKeyspaces(), result.schema.getKeyspaces());
+        clientWarnings(diff).forEach(ClientWarn.instance::warn);
 
-        if (result.diff.isEmpty())
+        if (diff.isEmpty())
             return new ResultMessage.Void();
 
         /*
@@ -127,9 +174,9 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
          */
         AuthenticatedUser user = state.getClientState().getUser();
         if (null != user && !user.isAnonymous())
-            createdResources(result.diff).forEach(r -> grantPermissionsOnResource(r, user));
+            createdResources(diff).forEach(r -> grantPermissionsOnResource(r, user));
 
-        return new ResultMessage.SchemaChange(schemaChangeEvent(result.diff));
+        return new ResultMessage.SchemaChange(schemaChangeEvent(diff));
     }
 
     private void validateKeyspaceName()
@@ -170,4 +217,12 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
     {
         return new InvalidRequestException(String.format(format, args));
     }
+
+    public String toString()
+    {
+        return "AlterSchemaStatement{" +
+               "keyspaceName='" + keyspaceName + '\'' +
+               ", cql='" + cql() + '\'' +
+               '}';
+    }
 }

src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java

@@ -24,14 +24,15 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Nullable;
 
 import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,38 +45,38 @@ import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.AbstractType;
-
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.index.TargetParser;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.MemtableParams;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.schema.Views;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
+import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.NoSpamLogger;
-
+import org.apache.cassandra.utils.Pair;
-import static java.lang.String.format;
-import static java.lang.String.join;
 
 import static com.google.common.collect.Iterables.isEmpty;
-import static com.google.common.collect.Iterables.transform;
+import static java.lang.String.format;
-
+import static java.lang.String.join;
 import static org.apache.cassandra.schema.TableMetadata.Flag;
 
 public abstract class AlterTableStatement extends AlterSchemaStatement
@@ -100,8 +101,9 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         this.state = state;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         TableMetadata table = null == keyspace
@@ -118,7 +120,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         if (table.isView())
             throw ire("Cannot use ALTER TABLE on a materialized view; use ALTER MATERIALIZED VIEW instead");
 
-        return schema.withAddedOrUpdated(apply(keyspace, table));
+        return schema.withAddedOrUpdated(apply(metadata.nextEpoch(), keyspace, table));
     }
 
     SchemaChange schemaChangeEvent(KeyspacesDiff diff)
@@ -142,7 +144,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         return format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, tableName);
     }
 
-    abstract KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table);
+    abstract KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table);
 
     /**
      * {@code ALTER TABLE [IF EXISTS] <table> ALTER <column> TYPE <newtype>;}
@@ -156,7 +158,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             super(keyspaceName, tableName, ifTableExists);
         }
 
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             throw ire("Altering column types is no longer supported");
         }
@@ -196,7 +198,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         }
 
         @Override
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             ColumnMetadata column = table.getColumn(columnName);
 
@@ -214,7 +216,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             if (Objects.equals(oldMask, newMask))
                 return keyspace;
 
-            TableMetadata.Builder tableBuilder = table.unbuild();
+            TableMetadata.Builder tableBuilder = table.unbuild().epoch(epoch);
             tableBuilder.alterColumnMask(columnName, newMask);
             TableMetadata newTable = tableBuilder.build();
             newTable.validate();
@@ -274,10 +276,10 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             newColumns.forEach(c -> c.type.validate(state, "Column " + c.name));
         }
 
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);
-            TableMetadata.Builder tableBuilder = table.unbuild();
+            TableMetadata.Builder tableBuilder = table.unbuild().epoch(epoch);
             Views.Builder viewsBuilder = keyspace.views.unbuild();
             newColumns.forEach(c -> addColumn(keyspace, table, c, ifColumnNotExists, tableBuilder, viewsBuilder));
 
@@ -360,6 +362,38 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         }
     }
 
+    private static void validateIndexesForColumnModification(TableMetadata table,
+                                                             ColumnIdentifier colId,
+                                                             boolean isRename)
+    {
+        ColumnMetadata column = table.getColumn(colId);
+        Set<String> dependentIndexes = new HashSet<>();
+        for (IndexMetadata index : table.indexes)
+        {
+            Optional<Pair<ColumnMetadata, IndexTarget.Type>> target = TargetParser.tryParse(table, index);
+            if (target.isEmpty())
+            {
+                // The target column(s) of this index is not trivially discernible from its metadata.
+                // This implies an external custom index implementation and without instantiating the
+                // index itself we cannot be sure that the column metadata is safe to modify.
+                dependentIndexes.add(index.name);
+            }
+            else if (target.get().left.equals(column))
+            {
+                // The index metadata declares an explicit dependency on the column being modified, so
+                // the mutation must be rejected.
+                dependentIndexes.add(index.name);
+            }
+        }
+        if (!dependentIndexes.isEmpty())
+        {
+            throw ire("Cannot %s column %s because it has dependent secondary indexes (%s)",
+                      isRename ? "rename" : "drop",
+                      colId,
+                      join(", ", dependentIndexes));
+        }
+    }
+
     /**
      * {@code ALTER TABLE [IF EXISTS] <table> DROP [IF EXISTS] <column>}
      * {@code ALTER TABLE [IF EXISTS] <table> DROP [IF EXISTS] ( <column>, <column1>, ... <columnn>)}
@@ -379,7 +413,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             this.timestamp = timestamp;
         }
 
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);
             TableMetadata.Builder builder = table.unbuild();
@@ -407,14 +441,8 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             if (currentColumn.type.isUDT() && currentColumn.type.isMultiCell())
                 throw ire("Cannot drop non-frozen column %s of user type %s", column, currentColumn.type.asCQL3Type());
 
-            // TODO: some day try and find a way to not rely on Keyspace/IndexManager/Index to find dependent indexes
+            if (!table.indexes.isEmpty())
-            Set<IndexMetadata> dependentIndexes = Keyspace.openAndGetStore(table).indexManager.getDependentIndexes(currentColumn);
+                AlterTableStatement.validateIndexesForColumnModification(table, column, false);
-            if (!dependentIndexes.isEmpty())
-            {
-                throw ire("Cannot drop column %s because it has dependent secondary indexes (%s)",
-                          currentColumn,
-                          join(", ", transform(dependentIndexes, i -> i.name)));
-            }
 
             if (!isEmpty(keyspace.views.forTable(table.id)))
                 throw ire("Cannot drop column %s on base table %s with materialized views", currentColumn, table.name);
@@ -447,10 +475,10 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             this.ifColumnsExists = ifColumnsExists;
         }
 
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             Guardrails.alterTableEnabled.ensureEnabled("ALTER TABLE changing columns", state);
-            TableMetadata.Builder tableBuilder = table.unbuild();
+            TableMetadata.Builder tableBuilder = table.unbuild().epoch(epoch);
             Views.Builder viewsBuilder = keyspace.views.unbuild();
             renamedColumns.forEach((o, n) -> renameColumn(keyspace, table, o, n, ifColumnsExists, tableBuilder, viewsBuilder));
 
@@ -485,14 +513,8 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
                           table);
             }
 
-            // TODO: some day try and find a way to not rely on Keyspace/IndexManager/Index to find dependent indexes
+            if (!table.indexes.isEmpty())
-            Set<IndexMetadata> dependentIndexes = Keyspace.openAndGetStore(table).indexManager.getDependentIndexes(column);
+                AlterTableStatement.validateIndexesForColumnModification(table, oldName, true);
-            if (!dependentIndexes.isEmpty())
-            {
-                throw ire("Can't rename column %s because it has dependent secondary indexes (%s)",
-                          oldName,
-                          join(", ", transform(dependentIndexes, i -> i.name)));
-            }
 
             for (ViewMetadata view : keyspace.views.forTable(table.id))
             {
@@ -523,13 +545,15 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         public void validate(ClientState state)
         {
             super.validate(state);
-
+            // If a memtable configuration is specified, validate it against config
+            if (attrs.hasOption(TableParams.Option.MEMTABLE))
+                MemtableParams.get(attrs.getString(TableParams.Option.MEMTABLE.toString()));
             Guardrails.tableProperties.guard(attrs.updatedProperties(), attrs::removeProperty, state);
 
             validateDefaultTimeToLive(attrs.asNewTableParams());
         }
 
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             attrs.validate();
 
@@ -547,7 +571,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
                           "before being replayed.");
             }
 
-            if (keyspace.createReplicationStrategy().hasTransientReplicas()
+            if (keyspace.replicationStrategy.hasTransientReplicas()
                 && params.readRepair != ReadRepairStrategy.NONE)
             {
                 throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces");
@@ -573,7 +597,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             super(keyspaceName, tableName, ifTableExists);
         }
 
-        public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
+        public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table)
         {
             if (!DatabaseDescriptor.enableDropCompactStorage())
                 throw new InvalidRequestException("DROP COMPACT STORAGE is disabled. Enable in cassandra.yaml to use.");
@@ -587,7 +611,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
                             ? ImmutableSet.of(Flag.COMPOUND, Flag.COUNTER)
                             : ImmutableSet.of(Flag.COMPOUND);
 
-            return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(flags)));
+            return keyspace.withSwapped(keyspace.tables.withSwapped(table.unbuild().flags(flags).build()));
         }
 
         /**
@@ -608,40 +632,45 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
             Set<InetAddressAndPort> preC15897nodes = new HashSet<>();
             Set<InetAddressAndPort> with2xSStables = new HashSet<>();
             Splitter onComma = Splitter.on(',').omitEmptyStrings().trimResults();
-            for (InetAddressAndPort node : StorageService.instance.getTokenMetadata().getAllEndpoints())
+            Directory directory = ClusterMetadata.current().directory;
+            for (InetAddressAndPort node : directory.allAddresses())
             {
-                if (MessagingService.instance().versions.knows(node) &&
+
-                    MessagingService.instance().versions.getRaw(node) < MessagingService.VERSION_40)
+                CassandraVersion version = directory.version(directory.peerId(node)).cassandraVersion;
+
+                if (version.compareTo(CassandraVersion.CASSANDRA_4_0) < 0)
                 {
+                    // if the cluster contains any pre-4.0 nodes (which really shouldn't be the case), reject this
+                    // operation as we can't be certain all peers can support it.
                     before4.add(node);
-                    continue;
                 }
-
+                else
-                String sstableVersionsString = Gossiper.instance.getApplicationState(node, ApplicationState.SSTABLE_VERSIONS);
-                if (sstableVersionsString == null)
                 {
-                    preC15897nodes.add(node);
+                    // any peer on a version greater than 4.0.0 must include CASSANDRA-15897, so just check that
-                    continue;
+                    // its min sstable version. Note: this app state may be empty/unset if the full StorageService
-                }
+                    // initialisation hasn't been done, i.e. in tests.
-
+                    String sstableVersionsString = Gossiper.instance.getApplicationState(node, ApplicationState.SSTABLE_VERSIONS);
-                try
+                    if (Strings.isNullOrEmpty(sstableVersionsString))
-                {
+                        continue;
-                    boolean has2xSStables = onComma.splitToList(sstableVersionsString)
+                    try
-                                                   .stream()
+                    {
-                                                   .anyMatch(v -> v.compareTo("big-ma")<=0);
+                        boolean has2xSStables = onComma.splitToList(sstableVersionsString)
-                    if (has2xSStables)
+                                                       .stream()
-                        with2xSStables.add(node);
+                                                       .anyMatch(v -> v.compareTo("big-ma")<=0);
-                }
+                        if (has2xSStables)
-                catch (IllegalArgumentException e)
+                            with2xSStables.add(node);
-                {
+                    }
-                    // Means VersionType::fromString didn't parse a version correctly. Which shouldn't happen, we shouldn't
+                    catch (IllegalArgumentException e)
-                    // have garbage in Gossip. But crashing the request is not ideal, so we log the error but ignore the
+                    {
-                    // node otherwise.
+                        // Means VersionType::fromString didn't parse a version correctly. Which shouldn't happen, we shouldn't
-                    noSpamLogger.error("Unexpected error parsing sstable versions from gossip for {} (gossiped value " +
+                        // have garbage in Gossip. But crashing the request is not ideal, so we log the error but ignore the
-                                       "is '{}'). This is a bug and should be reported. Cannot ensure that {} has no " +
+                        // node otherwise.
-                                       "non-upgraded 2.x sstables anymore. If after this DROP COMPACT STORAGE some old " +
+                        noSpamLogger.error("Unexpected error parsing sstable versions from gossip for {} (gossiped value " +
-                                       "sstables cannot be read anymore, please use `upgradesstables` with the " +
+                                           "is '{}'). This is a bug and should be reported. Cannot ensure that {} has no " +
-                                       "`--force-compact-storage-on` option.", node, sstableVersionsString, node);
+                                           "non-upgraded 2.x sstables anymore. If after this DROP COMPACT STORAGE some old " +
+                                           "sstables cannot be read anymore, please use `upgradesstables` with the " +
+                                           "`--force-compact-storage-on` option.", node, sstableVersionsString, node);
+                    }
                 }
             }
 
@@ -649,10 +678,6 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
                 throw new InvalidRequestException(format("Cannot DROP COMPACT STORAGE as some nodes in the cluster (%s) " +
                                                          "are not on 4.0+ yet. Please upgrade those nodes and run " +
                                                          "`upgradesstables` before retrying.", before4));
-            if (!preC15897nodes.isEmpty())
-                throw new InvalidRequestException(format("Cannot guarantee that DROP COMPACT STORAGE is safe as some nodes " +
-                                                         "in the cluster (%s) do not have https://issues.apache.org/jira/browse/CASSANDRA-15897. " +
-                                                         "Please upgrade those nodes and retry.", preC15897nodes));
             if (!with2xSStables.isEmpty())
                 throw new InvalidRequestException(format("Cannot DROP COMPACT STORAGE as some nodes in the cluster (%s) " +
                                                          "has some non-upgraded 2.x sstables. Please run `upgradesstables` " +

src/java/org/apache/cassandra/cql3/statements/schema/AlterTypeStatement.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -69,8 +70,10 @@ public abstract class AlterTypeStatement extends AlterSchemaStatement
         return new SchemaChange(Change.UPDATED, Target.TYPE, keyspaceName, typeName);
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         UserType type = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/AlterViewStatement.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -54,8 +55,10 @@ public final class AlterViewStatement extends AlterSchemaStatement
         this.state = state;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         ViewMetadata view = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java

@@ -44,6 +44,7 @@ import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -89,7 +90,8 @@ public final class CreateAggregateStatement extends AlterSchemaStatement
         this.ifNotExists = ifNotExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         if (ifNotExists && orReplace)
             throw ire("Cannot use both 'OR REPLACE' and 'IF NOT EXISTS' directives");
@@ -105,6 +107,7 @@ public final class CreateAggregateStatement extends AlterSchemaStatement
         if (!rawStateType.isImplicitlyFrozen() && rawStateType.isFrozen())
             throw ire("State type '%s' cannot be frozen; remove frozen<> modifier from '%s'", rawStateType, rawStateType);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);

src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -82,7 +83,7 @@ public final class CreateFunctionStatement extends AlterSchemaStatement
     }
 
     // TODO: replace affected aggregates !!
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         if (ifNotExists && orReplace)
             throw ire("Cannot use both 'OR REPLACE' and 'IF NOT EXISTS' directives");
@@ -103,6 +104,7 @@ public final class CreateFunctionStatement extends AlterSchemaStatement
         if (!rawReturnType.isImplicitlyFrozen() && rawReturnType.isFrozen())
             throw ire("Return type '%s' cannot be frozen; remove frozen<> modifier from '%s'", rawReturnType, rawReturnType);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);

src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java

@@ -31,7 +31,6 @@ import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget.Type;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.MapType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -40,6 +39,7 @@ import org.apache.cassandra.index.sasi.SASIIndex;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -112,7 +112,8 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         this.state = state;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         attrs.validate();
 
@@ -121,6 +122,7 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         if (attrs.isCustom && attrs.customClass.equals(SASIIndex.class.getName()) && !DatabaseDescriptor.getSASIIndexesEnabled())
             throw new InvalidRequestException(SASI_INDEX_DISABLED);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire(KEYSPACE_DOES_NOT_EXIST, keyspaceName);
@@ -143,7 +145,7 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         if (table.isView())
             throw ire(MATERIALIZED_VIEWS_NOT_SUPPORTED);
 
-        if (Keyspace.open(table.keyspace).getReplicationStrategy().hasTransientReplicas())
+        if (keyspace.replicationStrategy.hasTransientReplicas())
             throw new InvalidRequestException(TRANSIENTLY_REPLICATED_KEYSPACE_NOT_SUPPORTED);
 
         // guardrails to limit number of secondary indexes per table.

src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.HashSet;
 import java.util.Set;
 
 import com.google.common.collect.ImmutableSet;
@@ -36,12 +35,11 @@ import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
 import org.apache.cassandra.locator.LocalStrategy;
 import org.apache.cassandra.locator.SimpleStrategy;
-import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.KeyspaceParams.Option;
-import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
@@ -51,7 +49,6 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement
 
     private final KeyspaceAttributes attrs;
     private final boolean ifNotExists;
-    private final HashSet<String> clientWarnings = new HashSet<>();
 
     public CreateKeyspaceStatement(String keyspaceName, KeyspaceAttributes attrs, boolean ifNotExists)
     {
@@ -60,7 +57,7 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement
         this.ifNotExists = ifNotExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         attrs.validate();
 
@@ -70,6 +67,7 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement
         if (attrs.getReplicationStrategyClass() != null && attrs.getReplicationStrategyClass().equals(SimpleStrategy.class.getSimpleName()))
             Guardrails.simpleStrategyEnabled.ensureEnabled("SimpleStrategy", state);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         if (schema.containsKeyspace(keyspaceName))
         {
             if (ifNotExists)
@@ -83,7 +81,11 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement
         if (keyspace.params.replication.klass.equals(LocalStrategy.class))
             throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");
 
-        keyspace.params.validate(keyspaceName, state);
+        if (keyspace.params.replication.isMeta())
+            throw ire("Can not create a keyspace with MetaReplicationStrategy");
+
+        keyspace.params.validate(keyspaceName, state, metadata);
+        keyspace.replicationStrategy.validateExpectedOptions(metadata);
         return schema.withAddedOrUpdated(keyspace);
     }
 

src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java

@@ -34,15 +34,16 @@ import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.DataResource;
 import org.apache.cassandra.auth.IResource;
 import org.apache.cassandra.auth.Permission;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -94,8 +95,9 @@ public final class CreateTableStatement extends AlterSchemaStatement
         this.useCompactStorage = useCompactStorage;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);
@@ -108,10 +110,13 @@ public final class CreateTableStatement extends AlterSchemaStatement
             throw new AlreadyExistsException(keyspaceName, tableName);
         }
 
-        TableMetadata table = builder(keyspace.types).build();
+        TableMetadata.Builder builder = builder(keyspace.types).epoch(metadata.nextEpoch());
+        if (!builder.hasId() && !DatabaseDescriptor.useDeterministicTableID())
+            builder.id(TableId.get(metadata));
+        TableMetadata table = builder.build();
         table.validate();
 
-        if (keyspace.createReplicationStrategy().hasTransientReplicas()
+        if (keyspace.replicationStrategy.hasTransientReplicas()
             && table.params.readRepair != ReadRepairStrategy.NONE)
         {
             throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces");
@@ -128,6 +133,10 @@ public final class CreateTableStatement extends AlterSchemaStatement
     {
         super.validate(state);
 
+        // If a memtable configuration is specified, validate it against config
+        if (attrs.hasOption(TableParams.Option.MEMTABLE))
+            MemtableParams.get(attrs.getString(TableParams.Option.MEMTABLE.toString()));
+
         // Guardrail on table properties
         Guardrails.tableProperties.guard(attrs.updatedProperties(), attrs::removeProperty, state);
 
@@ -139,8 +148,7 @@ public final class CreateTableStatement extends AlterSchemaStatement
         {
             int totalUserTables = Schema.instance.getUserKeyspaces()
                                                  .stream()
-                                                 .map(Keyspace::open)
+                                                 .mapToInt(ksm -> ksm.tables.size())
-                                                 .mapToInt(keyspace -> keyspace.getColumnFamilyStores().size())
                                                  .sum();
             Guardrails.tables.guard(totalUserTables + 1, tableName, false, state);
         }

src/java/org/apache/cassandra/cql3/statements/schema/CreateTriggerStatement.java

@@ -24,6 +24,7 @@ import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.triggers.TriggerExecutor;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
@@ -45,8 +46,10 @@ public final class CreateTriggerStatement extends AlterSchemaStatement
         this.ifNotExists = ifNotExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);

src/java/org/apache/cassandra/cql3/statements/schema/CreateTypeStatement.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Types;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -75,8 +76,9 @@ public final class CreateTypeStatement extends AlterSchemaStatement
         }
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);

src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java

@@ -41,6 +41,7 @@ import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -110,7 +111,8 @@ public final class CreateViewStatement extends AlterSchemaStatement
         this.state = state;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         if (!DatabaseDescriptor.getMaterializedViewsEnabled())
             throw ire("Materialized views are disabled. Enable in cassandra.yaml to use.");
@@ -119,11 +121,12 @@ public final class CreateViewStatement extends AlterSchemaStatement
          * Basic dependency validations
          */
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);
 
-        if (keyspace.createReplicationStrategy().hasTransientReplicas())
+        if (keyspace.replicationStrategy.hasTransientReplicas())
             throw new InvalidRequestException("Materialized views are not supported on transiently replicated keyspaces");
 
         TableMetadata table = keyspace.tables.getNullable(tableName);
@@ -324,6 +327,8 @@ public final class CreateViewStatement extends AlterSchemaStatement
 
         if (attrs.hasProperty(TableAttributes.ID))
             builder.id(attrs.getId());
+        else if (!builder.hasId() && !DatabaseDescriptor.useDeterministicTableID())
+            builder.id(TableId.get(metadata));
 
         builder.params(attrs.asNewTableParams())
                .kind(TableMetadata.Kind.VIEW);

src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
@@ -64,13 +65,14 @@ public final class DropAggregateStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         String name =
             argumentsSpeficied
           ? format("%s.%s(%s)", keyspaceName, aggregateName, join(", ", transform(arguments, CQL3Type.Raw::toString)))
           : format("%s.%s", keyspaceName, aggregateName);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
         {

src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
@@ -65,13 +66,15 @@ public final class DropFunctionStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         String name =
             argumentsSpeficied
           ? format("%s.%s(%s)", keyspaceName, functionName, join(", ", transform(arguments, CQL3Type.Raw::toString)))
           : format("%s.%s", keyspaceName, functionName);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
         {

src/java/org/apache/cassandra/cql3/statements/schema/DropIndexStatement.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -42,8 +43,10 @@ public final class DropIndexStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         TableMetadata table = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/DropKeyspaceStatement.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 
@@ -38,10 +39,12 @@ public final class DropKeyspaceStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         Guardrails.dropKeyspaceEnabled.ensureEnabled(state);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         if (schema.containsKeyspace(keyspaceName))
             return schema.without(keyspaceName);
 

src/java/org/apache/cassandra/cql3/statements/schema/DropTableStatement.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -47,10 +48,11 @@ public final class DropTableStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         Guardrails.dropTruncateTableEnabled.ensureEnabled(state);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         TableMetadata table = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/DropTriggerStatement.java

@@ -24,6 +24,7 @@ import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -42,8 +43,10 @@ public final class DropTriggerStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         TableMetadata table = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/DropTypeStatement.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 import org.apache.cassandra.transport.Event.SchemaChange;
@@ -55,10 +56,12 @@ public final class DropTypeStatement extends AlterSchemaStatement
     }
 
     // TODO: expand types into tuples in all dropped columns of all tables
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
         ByteBuffer name = bytes(typeName);
 
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         UserType type = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/DropViewStatement.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -41,8 +42,10 @@ public final class DropViewStatement extends AlterSchemaStatement
         this.ifExists = ifExists;
     }
 
-    public Keyspaces apply(Keyspaces schema)
+    @Override
+    public Keyspaces apply(ClusterMetadata metadata)
     {
+        Keyspaces schema = metadata.schema.getKeyspaces();
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
         ViewMetadata view = null == keyspace

src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java

@@ -115,8 +115,8 @@ public final class TableAttributes extends PropertyDefinitions
         if (hasOption(COMPRESSION))
             builder.compression(CompressionParams.fromMap(getMap(COMPRESSION)));
 
-        if (hasOption(MEMTABLE))
+        if (hasOption(Option.MEMTABLE))
-            builder.memtable(MemtableParams.get(getString(MEMTABLE)));
+            builder.memtable(MemtableParams.getWithFallback(getString(Option.MEMTABLE)));
 
         if (hasOption(DEFAULT_TIME_TO_LIVE))
             builder.defaultTimeToLive(getInt(DEFAULT_TIME_TO_LIVE));

src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java

@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.exceptions.CoordinatorBehindException;
+import org.apache.cassandra.exceptions.InvalidRoutingException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.metrics.TCMMetrics;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+public abstract class AbstractMutationVerbHandler<T extends IMutation> implements IVerbHandler<T>
+{
+    private static final Logger logger = LoggerFactory.getLogger(AbstractMutationVerbHandler.class);
+    private static final String logMessageTemplate = "Received mutation from {} for token {} outside valid range for keyspace {}";
+
+    public void doVerb(Message<T> message) throws IOException
+    {
+        processMessage(message, message.respondTo());
+    }
+
+    protected void processMessage(Message<T> message, InetAddressAndPort respondTo)
+    {
+        if (message.epoch().isAfter(Epoch.EMPTY))
+        {
+            ClusterMetadata metadata = ClusterMetadata.current();
+            metadata = checkTokenOwnership(metadata, message);
+            metadata = checkSchemaVersion(metadata, message);
+        }
+        applyMutation(message, respondTo);
+    }
+
+    abstract void applyMutation(Message<T> message, InetAddressAndPort respondToAddress);
+
+    private ClusterMetadata checkTokenOwnership(ClusterMetadata metadata, Message<T> message)
+    {
+        String keyspace = message.payload.getKeyspaceName();
+        DecoratedKey key = message.payload.key();
+
+        VersionedEndpoints.ForToken forToken = writePlacements(metadata, keyspace, key);
+
+        if (message.epoch().isAfter(metadata.epoch))
+        {
+            // If replica detects that coordinator has made an out-of-range request, it has to catch up blockingly,
+            // since coordinator's routing may be more recent.
+            if (!forToken.get().containsSelf())
+            {
+                metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch());
+                forToken = writePlacements(metadata, keyspace, key);
+            }
+            // Otherwise, coordinator and the replica agree about the placement of the givent token, so catch-up can be async
+            else
+            {
+                ClusterMetadataService.instance().fetchLogFromPeerOrCMSAsync(metadata, message.from(), message.epoch());
+            }
+        }
+
+        if (!forToken.get().containsSelf())
+        {
+            StorageService.instance.incOutOfRangeOperationCount();
+            Keyspace.open(message.payload.getKeyspaceName()).metric.outOfRangeTokenWrites.inc();
+            NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, message.from(), key.getToken(), message.payload.getKeyspaceName());
+            throw InvalidRoutingException.forWrite(message.from(), key.getToken(), metadata.epoch, message.payload);
+        }
+
+        if (forToken.lastModified().isAfter(message.epoch()))
+        {
+            TCMMetrics.instance.coordinatorBehindPlacements.mark();
+            throw new CoordinatorBehindException(String.format("Routing is correct, but coordinator needs to catch-up at least to epoch %s to maintain consistency. Current coordinator epoch is %s",
+                                                               forToken.lastModified(), message.epoch()));
+        }
+
+        return metadata;
+    }
+
+    private ClusterMetadata checkSchemaVersion(ClusterMetadata metadata, Message<T> message)
+    {
+        if (SchemaConstants.isSystemKeyspace(message.payload.getKeyspaceName()) || message.epoch().is(metadata.epoch))
+            return metadata;
+        String keyspace = message.payload.getKeyspaceName();
+        Keyspace ks = metadata.schema.getKeyspace(keyspace);
+        if (ks != null)
+        {
+            if (message.epoch().isAfter(metadata.epoch))
+            {
+                // coordinator is ahead - check each partition update if the schema is ahead of the schema we have for the table
+                for (PartitionUpdate pu : message.payload.getPartitionUpdates())
+                {
+                    Epoch remoteSchemaEpoch = pu.serializedAtEpoch;
+                    if (remoteSchemaEpoch != null && remoteSchemaEpoch.isAfter(metadata.epoch))
+                    {
+                        // the partition update was serialized after the epoch we currently know, catch up and
+                        // make sure we've seen the epoch it has seen, otherwise fail request.
+                        metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch());
+                        if (pu.serializedAtEpoch.isAfter(metadata.epoch))
+                            throw new IllegalStateException(String.format("Coordinator %s is still ahead after fetching log, our epoch = %s, their epoch = %s",
+                                                                          message.from(),
+                                                                          metadata.epoch, message.epoch()));
+                    }
+                }
+            }
+            else if (message.epoch().isBefore(metadata.schema.lastModified()))
+            {
+                // coordinator might not have seen the latest schema change - check each modified table individually
+                for (PartitionUpdate pu : message.payload.getPartitionUpdates())
+                {
+                    // coordinator could be behind, check local tables
+                    ColumnFamilyStore cfs = ks.getColumnFamilyStore(pu.metadata().id);
+                    if (cfs != null)
+                    {
+                        Epoch remoteSchemaEpoch = pu.serializedAtEpoch;
+                        if (remoteSchemaEpoch != null && remoteSchemaEpoch.isBefore(cfs.metadata().epoch))
+                        {
+                            TCMMetrics.instance.coordinatorBehindSchema.mark();
+                            throw new CoordinatorBehindException(String.format("Coordinator %s is behind, our epoch = %s, their epoch = %s",
+                                                                               message.from(),
+                                                                               metadata.epoch, message.epoch()));
+                        }
+                    }
+                    else
+                    {
+                        TCMMetrics.instance.coordinatorBehindSchema.mark();
+                        throw new CoordinatorBehindException(String.format("Schema mismatch, coordinator %s is behind, we're missing table %s.%s, our epoch = %s, their epoch = %s",
+                                                                           message.from(),
+                                                                           pu.metadata().keyspace,
+                                                                           pu.metadata().name,
+                                                                           metadata.epoch, message.epoch()));
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (message.epoch().isBefore(metadata.schema.lastModified()))
+            {
+                TCMMetrics.instance.coordinatorBehindSchema.mark();
+                throw new CoordinatorBehindException(String.format("Schema mismatch, coordinator %s is behind, we're missing keyspace %s, our epoch = %s, their epoch = %s",
+                                                                   message.from(),
+                                                                   keyspace,
+                                                                   metadata.epoch, message.epoch()));
+            }
+            else
+            {
+                metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch());
+            }
+        }
+
+        return metadata;
+    }
+
+    private static VersionedEndpoints.ForToken writePlacements(ClusterMetadata metadata, String keyspace, DecoratedKey key)
+    {
+        return metadata.placements.get(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken());
+    }
+}

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -90,14 +90,14 @@ import org.apache.cassandra.db.compaction.CompactionStrategyManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.memtable.Flushing;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.ShardBoundaries;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.db.memtable.Flushing;
-import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.db.memtable.ShardBoundaries;
 import org.apache.cassandra.db.partitions.CachedPartition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.repair.CassandraTableRepairManager;
@@ -161,6 +161,8 @@ import org.apache.cassandra.service.snapshot.SnapshotLoader;
 import org.apache.cassandra.service.snapshot.SnapshotManifest;
 import org.apache.cassandra.service.snapshot.TableSnapshot;
 import org.apache.cassandra.streaming.TableStreamManager;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.DefaultValue;
 import org.apache.cassandra.utils.ExecutorUtils;
@@ -261,7 +263,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     static final String TOKEN_DELIMITER = ":";
 
     /** Special values used when the local ranges are not changed with ring changes (e.g. local tables). */
-    public static final int RING_VERSION_IRRELEVANT = -1;
+    // TODO - make this Epoch.EMPTY
+    public static final Epoch RING_VERSION_IRRELEVANT = Epoch.create(-1);
 
     static
     {
@@ -375,26 +378,31 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     }
 
     public void reload()
+    {
+        reload(metadata());
+    }
+
+    public void reload(TableMetadata tableMetadata)
     {
         // metadata object has been mutated directly. make all the members jibe with new settings.
 
         // only update these runtime-modifiable settings if they have not been modified.
         if (!minCompactionThreshold.isModified())
             for (ColumnFamilyStore cfs : concatWithIndexes())
-                cfs.minCompactionThreshold = new DefaultValue(metadata().params.compaction.minCompactionThreshold());
+                cfs.minCompactionThreshold = new DefaultValue<>(tableMetadata.params.compaction.minCompactionThreshold());
         if (!maxCompactionThreshold.isModified())
             for (ColumnFamilyStore cfs : concatWithIndexes())
-                cfs.maxCompactionThreshold = new DefaultValue(metadata().params.compaction.maxCompactionThreshold());
+                cfs.maxCompactionThreshold = new DefaultValue<>(tableMetadata.params.compaction.maxCompactionThreshold());
         if (!crcCheckChance.isModified())
             for (ColumnFamilyStore cfs : concatWithIndexes())
-                cfs.crcCheckChance = new DefaultValue(metadata().params.crcCheckChance);
+                cfs.crcCheckChance = new DefaultValue<>(tableMetadata.params.crcCheckChance);
 
-        compactionStrategyManager.maybeReloadParamsFromSchema(metadata().params.compaction);
+        compactionStrategyManager.maybeReloadParamsFromSchema(tableMetadata.params.compaction);
 
-        indexManager.reload();
+        indexManager.reload(tableMetadata);
 
-        memtableFactory = metadata().params.memtable.factory();
+        memtableFactory = tableMetadata.params.memtable.factory();
-        switchMemtableOrNotify(FlushReason.SCHEMA_CHANGE, Memtable::metadataUpdated);
+        switchMemtableOrNotify(FlushReason.SCHEMA_CHANGE, tableMetadata, Memtable::metadataUpdated);
     }
 
     public static Runnable getBackgroundCompactionTaskSubmitter()
@@ -453,7 +461,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         {
             CompressionParams params = CompressionParams.fromMap(opts);
             params.validate();
-            metadata.setLocalOverrides(metadata().unbuild().compression(params).build());
+
+            TableMetadata orig = metadata();
+            metadata.setLocalOverrides(orig.unbuild().compression(params).epoch(orig.epoch).build());
         }
         catch (ConfigurationException e)
         {
@@ -470,27 +480,26 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     public ColumnFamilyStore(Keyspace keyspace,
                              String columnFamilyName,
                              Supplier<? extends SSTableId> sstableIdGenerator,
-                             TableMetadataRef metadata,
+                             TableMetadata initMetadata,
                              Directories directories,
                              boolean loadSSTables,
-                             boolean registerBookeeping,
+                             boolean registerBookeeping)
-                             boolean offline)
     {
         assert directories != null;
-        assert metadata != null : "null metadata for " + keyspace + ':' + columnFamilyName;
+        assert initMetadata != null : "null metadata for " + keyspace + ':' + columnFamilyName;
 
         this.keyspace = keyspace;
-        this.metadata = metadata;
+        this.metadata = initMetadata.ref;
         this.directories = directories;
         name = columnFamilyName;
-        minCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.minCompactionThreshold());
+        minCompactionThreshold = new DefaultValue<>(initMetadata.params.compaction.minCompactionThreshold());
-        maxCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.maxCompactionThreshold());
+        maxCompactionThreshold = new DefaultValue<>(initMetadata.params.compaction.maxCompactionThreshold());
-        crcCheckChance = new DefaultValue<>(metadata.get().params.crcCheckChance);
+        crcCheckChance = new DefaultValue<>(initMetadata.params.crcCheckChance);
-        viewManager = keyspace.viewManager.forTable(metadata.id);
+        viewManager = keyspace.viewManager.forTable(initMetadata);
         this.sstableIdGenerator = sstableIdGenerator;
         sampleReadLatencyMicros = DatabaseDescriptor.getReadRpcTimeout(TimeUnit.MICROSECONDS) / 2;
         additionalWriteLatencyMicros = DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MICROSECONDS) / 2;
-        memtableFactory = metadata.get().params.memtable.factory();
+        memtableFactory = initMetadata.params.memtable.factory();
 
         logger.info("Initializing {}.{}", getKeyspaceName(), name);
 
@@ -514,7 +523,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         {
             Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
             sstables = SSTableReader.openAll(this, sstableFiles.list().entrySet(), metadata);
-            data.addInitialSSTablesWithoutUpdatingSize(sstables);
+            data.addInitialSSTablesWithoutUpdatingSize(sstables, this);
         }
 
         // compaction strategy should be created after the CFS has been prepared
@@ -528,7 +537,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
         // create the private ColumnFamilyStores for the secondary column indexes
         indexManager = new SecondaryIndexManager(this);
-        for (IndexMetadata info : metadata.get().indexes)
+        for (IndexMetadata info : initMetadata.indexes)
         {
             indexManager.addIndex(info, true);
         }
@@ -563,7 +572,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         if (DatabaseDescriptor.isClientOrToolInitialized() || SchemaConstants.isSystemKeyspace(getKeyspaceName()))
             topPartitions = null;
         else
-            topPartitions = new TopPartitionTracker(metadata());
+            topPartitions = new TopPartitionTracker(initMetadata);
     }
 
     public static String getTableMBeanName(String ks, String name, boolean isIndex)
@@ -741,32 +750,31 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     }
 
 
-    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, TableMetadataRef metadata, boolean loadSSTables)
+    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, TableMetadata metadata, boolean loadSSTables)
     {
         return createColumnFamilyStore(keyspace, metadata.name, metadata, loadSSTables);
     }
 
     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,
-                                                                         String columnFamily,
+                                                            String columnFamily,
-                                                                         TableMetadataRef metadata,
+                                                            TableMetadata metadata,
-                                                                         boolean loadSSTables)
+                                                            boolean loadSSTables)
     {
-        Directories directories = new Directories(metadata.get());
+        Directories directories = new Directories(metadata);
-        return createColumnFamilyStore(keyspace, columnFamily, metadata, directories, loadSSTables, true, false);
+        return createColumnFamilyStore(keyspace, columnFamily, metadata, directories, loadSSTables, true);
     }
 
     /** This is only directly used by offline tools */
     public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,
                                                                          String columnFamily,
-                                                                         TableMetadataRef metadata,
+                                                                         TableMetadata metadata,
                                                                          Directories directories,
                                                                          boolean loadSSTables,
-                                                                         boolean registerBookkeeping,
+                                                                         boolean registerBookkeeping)
-                                                                         boolean offline)
     {
         return new ColumnFamilyStore(keyspace, columnFamily,
                                      directories.getUIDGenerator(SSTableIdFactory.instance.defaultBuilder()),
-                                     metadata, directories, loadSSTables, registerBookkeeping, offline);
+                                     metadata, directories, loadSSTables, registerBookkeeping);
     }
 
     /**
@@ -963,10 +971,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
      * Checks with the memtable if it should be switched for the given reason, and if not, calls the specified
      * notification method.
      */
-    private void switchMemtableOrNotify(FlushReason reason, Consumer<Memtable> elseNotify)
+    private void switchMemtableOrNotify(FlushReason reason, TableMetadata metadata, Consumer<Memtable> elseNotify)
     {
         Memtable currentMemtable = data.getView().getCurrentMemtable();
-        if (currentMemtable.shouldSwitch(reason))
+        if (currentMemtable.shouldSwitch(reason, metadata))
             switchMemtableIfCurrent(currentMemtable, reason);
         else
             elseNotify.accept(currentMemtable);
@@ -1476,9 +1484,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
     public static class VersionedLocalRanges extends ArrayList<Splitter.WeightedRange>
     {
-        public final long ringVersion;
+        public final Epoch ringVersion;
 
-        public VersionedLocalRanges(long ringVersion, int initialSize)
+        public VersionedLocalRanges(Epoch ringVersion, int initialSize)
         {
             super(initialSize);
             this.ringVersion = ringVersion;
@@ -1487,16 +1495,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
     public VersionedLocalRanges localRangesWeighted()
     {
+        ClusterMetadata metadata = ClusterMetadata.current();
         if (!SchemaConstants.isLocalSystemKeyspace(getKeyspaceName())
-            && getPartitioner() == StorageService.instance.getTokenMetadata().partitioner)
+            && getPartitioner() == metadata.partitioner)
         {
             DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
             Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
-            long ringVersion = versionedLocalRanges.ringVersion;
+            Epoch epoch = versionedLocalRanges.epoch;
-
             if (!localRanges.isEmpty())
             {
-                VersionedLocalRanges weightedRanges = new VersionedLocalRanges(ringVersion, localRanges.size());
+                VersionedLocalRanges weightedRanges = new VersionedLocalRanges(epoch, localRanges.size());
                 for (Range<Token> r : localRanges)
                 {
                     // WeightedRange supports only unwrapped ranges as it relies
@@ -1509,7 +1517,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
             }
             else
             {
-                return fullWeightedRange(ringVersion, getPartitioner());
+                return fullWeightedRange(epoch, getPartitioner());
             }
         }
         else
@@ -1527,11 +1535,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
             return ShardBoundaries.NONE;
 
         ShardBoundaries shardBoundaries = cachedShardBoundaries;
+        ClusterMetadata metadata = ClusterMetadata.currentNullable();
+        if (metadata == null)
+            return ShardBoundaries.NONE;
 
         if (shardBoundaries == null ||
             shardBoundaries.shardCount() != shardCount ||
-            (shardBoundaries.ringVersion != RING_VERSION_IRRELEVANT &&
+            (!shardBoundaries.epoch.equals(Epoch.EMPTY) && !shardBoundaries.epoch.equals(metadata.epoch)))
-             shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion()))
         {
             VersionedLocalRanges weightedRanges = localRangesWeighted();
 
@@ -1545,9 +1555,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     }
 
     @VisibleForTesting
-    public static VersionedLocalRanges fullWeightedRange(long ringVersion, IPartitioner partitioner)
+    public static VersionedLocalRanges fullWeightedRange(Epoch epoch, IPartitioner partitioner)
     {
-        VersionedLocalRanges ranges = new VersionedLocalRanges(ringVersion, 1);
+        VersionedLocalRanges ranges = new VersionedLocalRanges(epoch, 1);
         ranges.add(new Splitter.WeightedRange(1.0, new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken())));
         return ranges;
     }
@@ -3341,14 +3351,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
     public DiskBoundaries getDiskBoundaries()
     {
-        return diskBoundaryManager.getDiskBoundaries(this);
+        return diskBoundaryManager.getDiskBoundaries(this, metadata.get());
+    }
+
+    public DiskBoundaries getDiskBoundaries(TableMetadata initialMetadata)
+    {
+        return diskBoundaryManager.getDiskBoundaries(this, initialMetadata);
     }
 
     public void invalidateLocalRanges()
     {
         diskBoundaryManager.invalidate();
 
-        switchMemtableOrNotify(FlushReason.OWNED_RANGES_CHANGE, Memtable::localRangesUpdated);
+        switchMemtableOrNotify(FlushReason.OWNED_RANGES_CHANGE, metadata(), Memtable::localRangesUpdated);
     }
 
     @Override

src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java

@@ -21,20 +21,20 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.StorageProxy;
 
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
-public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
+public class CounterMutationVerbHandler extends AbstractMutationVerbHandler<CounterMutation>
 {
     public static final CounterMutationVerbHandler instance = new CounterMutationVerbHandler();
 
     private static final Logger logger = LoggerFactory.getLogger(CounterMutationVerbHandler.class);
 
-    public void doVerb(final Message<CounterMutation> message)
+    protected void applyMutation(final Message<CounterMutation> message, InetAddressAndPort respondToAddress)
     {
         long queryStartNanoTime = nanoTime();
         final CounterMutation cm = message.payload;
@@ -50,7 +50,7 @@ public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
         // it's own in that case.
         StorageProxy.applyCounterMutationOnLeader(cm,
                                                   localDataCenter,
-                                                  () -> MessagingService.instance().send(message.emptyResponse(), message.from()),
+                                                  () -> MessagingService.instance().send(message.emptyResponse(), respondToAddress),
                                                   queryStartNanoTime);
     }
 }

src/java/org/apache/cassandra/db/DiskBoundaries.java

@@ -27,28 +27,32 @@ import com.google.common.collect.ImmutableList;
 
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.Epoch;
 
 public class DiskBoundaries
 {
     public final List<Directories.DataDirectory> directories;
     public final ImmutableList<PartitionPosition> positions;
-    final long ringVersion;
+    final Epoch epoch;
     final int directoriesVersion;
     private final ColumnFamilyStore cfs;
     private volatile boolean isInvalid = false;
 
     public DiskBoundaries(ColumnFamilyStore cfs, Directories.DataDirectory[] directories, int diskVersion)
     {
-        this(cfs, directories, null, -1, diskVersion);
+        this(cfs, directories, null, Epoch.EMPTY, diskVersion);
     }
 
     @VisibleForTesting
-    public DiskBoundaries(ColumnFamilyStore cfs, Directories.DataDirectory[] directories, List<PartitionPosition> positions, long ringVersion, int diskVersion)
+    public DiskBoundaries(ColumnFamilyStore cfs,
+                          Directories.DataDirectory[] directories,
+                          List<PartitionPosition> positions,
+                          Epoch epoch,
+                          int diskVersion)
     {
         this.directories = directories == null ? null : ImmutableList.copyOf(directories);
         this.positions = positions == null ? null : ImmutableList.copyOf(positions);
-        this.ringVersion = ringVersion;
+        this.epoch = epoch;
         this.directoriesVersion = diskVersion;
         this.cfs = cfs;
     }
@@ -60,7 +64,7 @@ public class DiskBoundaries
 
         DiskBoundaries that = (DiskBoundaries) o;
 
-        if (ringVersion != that.ringVersion) return false;
+        if (!epoch.equals(that.epoch)) return false;
         if (directoriesVersion != that.directoriesVersion) return false;
         if (!directories.equals(that.directories)) return false;
         return positions != null ? positions.equals(that.positions) : that.positions == null;
@@ -70,7 +74,7 @@ public class DiskBoundaries
     {
         int result = directories != null ? directories.hashCode() : 0;
         result = 31 * result + (positions != null ? positions.hashCode() : 0);
-        result = 31 * result + (int) (ringVersion ^ (ringVersion >>> 32));
+        result = 31 * result + epoch.hashCode();
         result = 31 * result + directoriesVersion;
         return result;
     }
@@ -80,7 +84,7 @@ public class DiskBoundaries
         return "DiskBoundaries{" +
                "directories=" + directories +
                ", positions=" + positions +
-               ", ringVersion=" + ringVersion +
+               ", epoch=" + epoch +
                ", directoriesVersion=" + directoriesVersion +
                '}';
     }
@@ -93,8 +97,7 @@ public class DiskBoundaries
         if (isInvalid)
             return true;
         int currentDiskVersion = DisallowedDirectories.getDirectoriesVersion();
-        long currentRingVersion = StorageService.instance.getTokenMetadata().getRingVersion();
+        return currentDiskVersion != directoriesVersion;
-        return currentDiskVersion != directoriesVersion || (ringVersion != -1 && currentRingVersion != ringVersion);
     }
 
     public void invalidate()

src/java/org/apache/cassandra/db/DiskBoundaryManager.java

@@ -31,9 +31,12 @@ import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.RangesAtEndpoint;
-import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
-import org.apache.cassandra.service.PendingRangeCalculatorService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.utils.FBUtilities;
 
 public class DiskBoundaryManager
@@ -43,18 +46,24 @@ public class DiskBoundaryManager
 
     public DiskBoundaries getDiskBoundaries(ColumnFamilyStore cfs)
     {
-        if (!cfs.getPartitioner().splitter().isPresent())
+        return getDiskBoundaries(cfs, cfs.metadata());
+    }
+
+    public DiskBoundaries getDiskBoundaries(ColumnFamilyStore cfs, TableMetadata metadata)
+    {
+        if (!metadata.partitioner.splitter().isPresent())
             return new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(), DisallowedDirectories.getDirectoriesVersion());
+
         if (diskBoundaries == null || diskBoundaries.isOutOfDate())
         {
             synchronized (this)
             {
                 if (diskBoundaries == null || diskBoundaries.isOutOfDate())
                 {
-                    logger.debug("Refreshing disk boundary cache for {}.{}", cfs.getKeyspaceName(), cfs.getTableName());
+                    logger.trace("Refreshing disk boundary cache for {}.{}", cfs.getKeyspaceName(), cfs.getTableName());
                     DiskBoundaries oldBoundaries = diskBoundaries;
-                    diskBoundaries = getDiskBoundaryValue(cfs);
+                    diskBoundaries = getDiskBoundaryValue(cfs, metadata.partitioner);
-                    logger.debug("Updating boundaries from {} to {} for {}.{}", oldBoundaries, diskBoundaries, cfs.getKeyspaceName(), cfs.getTableName());
+                    logger.trace("Updating boundaries from {} to {} for {}.{}", oldBoundaries, diskBoundaries, cfs.getKeyspaceName(), cfs.getTableName());
                 }
             }
         }
@@ -70,12 +79,12 @@ public class DiskBoundaryManager
     static class VersionedRangesAtEndpoint
     {
         public final RangesAtEndpoint rangesAtEndpoint;
-        public final long ringVersion;
+        public final Epoch epoch;
 
-        VersionedRangesAtEndpoint(RangesAtEndpoint rangesAtEndpoint, long ringVersion)
+        VersionedRangesAtEndpoint(RangesAtEndpoint rangesAtEndpoint, Epoch epoch)
         {
             this.rangesAtEndpoint = rangesAtEndpoint;
-            this.ringVersion = ringVersion;
+            this.epoch = epoch;
         }
     }
 
@@ -83,26 +92,35 @@ public class DiskBoundaryManager
     {
         RangesAtEndpoint localRanges;
 
-        long ringVersion;
+        Epoch epoch;
-        TokenMetadata tmd;
+        ClusterMetadata metadata;
         do
         {
-            tmd = StorageService.instance.getTokenMetadata();
+            metadata = ClusterMetadata.current();
-            ringVersion = tmd.getRingVersion();
+            epoch = metadata.epoch;
-            localRanges = getLocalRanges(cfs, tmd);
+            localRanges = getLocalRanges(cfs, metadata);
-            logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion);
+            logger.debug("Got local ranges {} (epoch = {})", localRanges, epoch);
         }
-        while (ringVersion != tmd.getRingVersion()); // if ringVersion is different here it means that
+        while (!metadata.epoch.equals(ClusterMetadata.current().epoch)); // if epoch is different here it means that
-        // it might have changed before we calculated localRanges - recalculate
+                                                                         // it might have changed before we calculated localRanges - recalculate
-
+        return new VersionedRangesAtEndpoint(localRanges, epoch);
-        return new VersionedRangesAtEndpoint(localRanges, ringVersion);
     }
 
-    private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs)
+    private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs, IPartitioner partitioner)
     {
-        VersionedRangesAtEndpoint rangesAtEndpoint = getVersionedLocalRanges(cfs);
+        if (ClusterMetadataService.instance() == null)
-        RangesAtEndpoint localRanges = rangesAtEndpoint.rangesAtEndpoint;
+            return new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(), null, Epoch.EMPTY, DisallowedDirectories.getDirectoriesVersion());
-        long ringVersion = rangesAtEndpoint.ringVersion;
+
+        RangesAtEndpoint localRanges;
+
+        ClusterMetadata metadata;
+        do
+        {
+            metadata = ClusterMetadata.current();
+            localRanges = getLocalRanges(cfs, metadata);
+            logger.debug("Got local ranges {} (epoch = {})", localRanges, metadata.epoch);
+        }
+        while (metadata.epoch != ClusterMetadata.current().epoch);
 
         int directoriesVersion;
         Directories.DataDirectory[] dirs;
@@ -114,29 +132,31 @@ public class DiskBoundaryManager
         while (directoriesVersion != DisallowedDirectories.getDirectoriesVersion()); // if directoriesVersion has changed we need to recalculate
 
         if (localRanges == null || localRanges.isEmpty())
-            return new DiskBoundaries(cfs, dirs, null, ringVersion, directoriesVersion);
+            return new DiskBoundaries(cfs, dirs, null, metadata.epoch, directoriesVersion);
 
-        List<PartitionPosition> positions = getDiskBoundaries(localRanges, cfs.getPartitioner(), dirs);
+        List<PartitionPosition> positions = getDiskBoundaries(localRanges, partitioner, dirs);
 
-        return new DiskBoundaries(cfs, dirs, positions, ringVersion, directoriesVersion);
+        return new DiskBoundaries(cfs, dirs, positions, metadata.epoch, directoriesVersion);
     }
 
-    private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, TokenMetadata tmd)
+
+    private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, ClusterMetadata metadata)
     {
         RangesAtEndpoint localRanges;
+        DataPlacement placement;
         if (StorageService.instance.isBootstrapMode()
-        && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
+            && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
         {
-            PendingRangeCalculatorService.instance.blockUntilFinished();
+            placement = metadata.placements.get(cfs.keyspace.getMetadata().params.replication);
-            localRanges = tmd.getPendingRanges(cfs.getKeyspaceName(), FBUtilities.getBroadcastAddressAndPort());
         }
         else
         {
-            // Reason we use use the future settled TMD is that if we decommission a node, we want to stream
+            // Reason we use use the future settled metadata is that if we decommission a node, we want to stream
             // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
             // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
-            localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort());
+            placement = metadata.writePlacementAllSettled(cfs.keyspace.getMetadata());
         }
+        localRanges = placement.writes.byEndpoint().get(FBUtilities.getBroadcastAddressAndPort());
         return localRanges;
     }
 

src/java/org/apache/cassandra/db/Keyspace.java

@@ -56,13 +56,11 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.metrics.KeyspaceMetrics;
 import org.apache.cassandra.repair.KeyspaceRepairManager;
 import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.snapshot.TableSnapshot;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -91,6 +89,7 @@ public class Keyspace
     private static int TEST_FAIL_MV_LOCKS_COUNT = CassandraRelevantProperties.TEST_FAIL_MV_LOCKS_COUNT.getInt();
 
     public final KeyspaceMetrics metric;
+    public final KeyspaceMetadataRef metadataRef;
 
     // It is possible to call Keyspace.open without a running daemon, so it makes sense to ensure
     // proper directories here as well as in CassandraDaemon.
@@ -100,8 +99,6 @@ public class Keyspace
             DatabaseDescriptor.createAllDirectories();
     }
 
-    private volatile KeyspaceMetadata metadata;
-
     //OpOrder is defined globally since we need to order writes across
     //Keyspaces in the case of Views (batchlog of view mutations)
     public static final OpOrder writeOrder = new OpOrder();
@@ -109,12 +106,11 @@ public class Keyspace
     /* ColumnFamilyStore per column family */
     private final ConcurrentMap<TableId, ColumnFamilyStore> columnFamilyStores = new ConcurrentHashMap<>();
 
-    private volatile AbstractReplicationStrategy replicationStrategy;
     public final ViewManager viewManager;
     private final KeyspaceWriteHandler writeHandler;
-    private volatile ReplicationParams replicationParams;
     private final KeyspaceRepairManager repairManager;
     private final SchemaProvider schema;
+    private final String name;
 
     private static volatile boolean initialized = false;
 
@@ -148,23 +144,15 @@ public class Keyspace
     public static Keyspace open(String keyspaceName)
     {
         assert initialized || SchemaConstants.isLocalSystemKeyspace(keyspaceName) : "Initialized: " + initialized;
-        return open(keyspaceName, Schema.instance, true);
+        Keyspace ks = Schema.instance.getKeyspaceInstance(keyspaceName);
+        assert ks != null : "Unknown keyspace " + keyspaceName;
+        return ks;
     }
 
     // to only be used by org.apache.cassandra.tools.Standalone* classes
     public static Keyspace openWithoutSSTables(String keyspaceName)
     {
-        return open(keyspaceName, Schema.instance, false);
+        return Schema.instance.getKeyspaceInstance(keyspaceName);
-    }
-
-    public static Keyspace open(String keyspaceName, SchemaProvider schema, boolean loadSSTables)
-    {
-        return schema.maybeAddKeyspaceInstance(keyspaceName, () -> new Keyspace(keyspaceName, schema, loadSSTables));
-    }
-
-    public static ColumnFamilyStore openAndGetStore(TableMetadataRef tableRef)
-    {
-        return open(tableRef.keyspace).getColumnFamilyStore(tableRef.id);
     }
 
     public static ColumnFamilyStore openAndGetStore(TableMetadata table)
@@ -188,15 +176,9 @@ public class Keyspace
         }
     }
 
-    public void setMetadata(KeyspaceMetadata metadata)
-    {
-        this.metadata = metadata;
-        createReplicationStrategy(metadata);
-    }
-
     public KeyspaceMetadata getMetadata()
     {
-        return metadata;
+        return metadataRef.get();
     }
 
     public Collection<ColumnFamilyStore> getColumnFamilyStores()
@@ -216,7 +198,7 @@ public class Keyspace
     {
         ColumnFamilyStore cfs = columnFamilyStores.get(id);
         if (cfs == null)
-            throw new IllegalArgumentException("Unknown CF " + id);
+            throw new IllegalArgumentException(String.format("Unknown CF %s %s", id, columnFamilyStores));
         return cfs;
     }
 
@@ -317,38 +299,53 @@ public class Keyspace
         return getColumnFamilyStores().stream().flatMap(cfs -> cfs.listSnapshots().values().stream());
     }
 
+    public static Keyspace forSchema(String keyspaceName, SchemaProvider schema)
+    {
+        return new Keyspace(keyspaceName, schema, true);
+    }
+
     private Keyspace(String keyspaceName, SchemaProvider schema, boolean loadSSTables)
+    {
+        this(schema,  schema.getKeyspaceMetadata(keyspaceName), loadSSTables);
+    }
+
+    public Keyspace(SchemaProvider schema, KeyspaceMetadata metadata, boolean loadSSTables)
     {
         this.schema = schema;
-        metadata = schema.getKeyspaceMetadata(keyspaceName);
+        this.name = metadata.name;
-        assert metadata != null : "Unknown keyspace " + keyspaceName;
+
+        assert metadata != null : "Unknown keyspace " + metadata.name;
 
         if (metadata.isVirtual())
-            throw new IllegalStateException("Cannot initialize Keyspace with virtual metadata " + keyspaceName);
+            throw new IllegalStateException("Cannot initialize Keyspace with virtual metadata " + metadata.name);
-        createReplicationStrategy(metadata);
 
         this.metric = new KeyspaceMetrics(this);
         this.viewManager = new ViewManager(this);
+
+        this.metadataRef = new KeyspaceMetadataRef(metadata, schema);
         for (TableMetadata cfm : metadata.tablesAndViews())
         {
             logger.trace("Initializing {}.{}", getName(), cfm.name);
-            initCf(schema.getTableMetadataRef(cfm.id), loadSSTables);
+            initCf(cfm, loadSSTables);
         }
-        this.viewManager.reload(false);
+
+        this.viewManager.reload(metadata);
+        this.metadataRef.unsetInitial();
 
         this.repairManager = new CassandraKeyspaceRepairManager(this);
         this.writeHandler = new CassandraKeyspaceWriteHandler(this);
     }
 
-    private Keyspace(KeyspaceMetadata metadata)
+    public Keyspace(KeyspaceMetadata metadata)
     {
         this.schema = Schema.instance;
-        this.metadata = metadata;
+        this.name = metadata.name;
-        createReplicationStrategy(metadata);
+
         this.metric = new KeyspaceMetrics(this);
         this.viewManager = new ViewManager(this);
         this.repairManager = new CassandraKeyspaceRepairManager(this);
         this.writeHandler = new CassandraKeyspaceWriteHandler(this);
+        this.metadataRef = new KeyspaceMetadataRef(metadata, schema);
     }
 
     public KeyspaceRepairManager getRepairManager()
@@ -361,18 +358,6 @@ public class Keyspace
         return new Keyspace(metadata);
     }
 
-    private void createReplicationStrategy(KeyspaceMetadata ksm)
-    {
-        logger.info("Creating replication strategy " + ksm.name + " params " + ksm.params);
-        replicationStrategy = ksm.createReplicationStrategy();
-        if (!ksm.params.replication.equals(replicationParams))
-        {
-            logger.debug("New replication settings for keyspace {} - invalidating disk boundary caches", ksm.name);
-            columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateLocalRanges);
-        }
-        replicationParams = ksm.params.replication;
-    }
-
     // best invoked on the compaction manager.
     public void dropCf(TableId tableId, boolean dropData)
     {
@@ -433,7 +418,7 @@ public class Keyspace
     /**
      * adds a cf to internal structures, ends up creating disk files).
      */
-    public void initCf(TableMetadataRef metadata, boolean loadSSTables)
+    public void initCf(TableMetadata metadata, boolean loadSSTables)
     {
         ColumnFamilyStore cfs = columnFamilyStores.get(metadata.id);
 
@@ -442,7 +427,8 @@ public class Keyspace
             // CFS being created for the first time, either on server startup or new CF being added.
             // We don't worry about races here; startup is safe, and adding multiple idential CFs
             // simultaneously is a "don't do that" scenario.
-            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.id, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
+            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.id,
+                                                                      ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
             // CFS mbean instantiation will error out before we hit this, but in case that changes...
             if (oldCfs != null)
                 throw new IllegalStateException("added multiple mappings for cf id " + metadata.id);
@@ -452,7 +438,7 @@ public class Keyspace
             // re-initializing an existing CF.  This will happen if you cleared the schema
             // on this node and it's getting repopulated from the rest of the cluster.
             assert cfs.name.equals(metadata.name);
-            cfs.reload();
+            cfs.reload(metadata);
         }
     }
 
@@ -514,7 +500,7 @@ public class Keyspace
                                                boolean isDeferrable,
                                                Promise<?> future)
     {
-        if (TEST_FAIL_WRITES && metadata.name.equals(TEST_FAIL_WRITES_KS))
+        if (TEST_FAIL_WRITES && getMetadata().name.equals(TEST_FAIL_WRITES_KS))
             throw new RuntimeException("Testing write failures");
 
         Lock[] locks = null;
@@ -626,7 +612,7 @@ public class Keyspace
                     try
                     {
                         Tracing.trace("Creating materialized view mutations from base table replica");
-                        viewManager.forTable(upd.metadata().id).pushViewReplicaUpdates(upd, makeDurable, baseComplete);
+                        viewManager.forTable(upd.metadata()).pushViewReplicaUpdates(upd, makeDurable, baseComplete);
                     }
                     catch (Throwable t)
                     {
@@ -661,7 +647,7 @@ public class Keyspace
 
     public AbstractReplicationStrategy getReplicationStrategy()
     {
-        return replicationStrategy;
+        return getMetadata().replicationStrategy;
     }
 
     public List<Future<?>> flush(ColumnFamilyStore.FlushReason reason)
@@ -749,6 +735,11 @@ public class Keyspace
         return Schema.instance.getKeyspaces().stream().map(Schema.instance::getKeyspaceInstance).filter(Objects::nonNull);
     }
 
+    public static Iterable<Keyspace> nonSystem()
+    {
+        return Iterables.transform(Schema.instance.distributedKeyspaces().names(), Keyspace::open);
+    }
+
     public static Iterable<Keyspace> nonLocalStrategy()
     {
         return Iterables.transform(Schema.instance.distributedKeyspaces().names(), Keyspace::open);
@@ -767,6 +758,37 @@ public class Keyspace
 
     public String getName()
     {
-        return metadata.name;
+        return name;
+    }
+
+    private static class KeyspaceMetadataRef
+    {
+        // We need "initial" keyspace metadata for initCF to run, due to circular dependency
+        // between keyspace keyspace -> column family -> keyspace metadata. There are some
+        // calls within initCF that try accessing keyspace metadata, which requires the metadata
+        // of initializing keyspace to already be visible via ClusterMetadata#schema.
+        private KeyspaceMetadata initial;
+
+        private final String name;
+        private final SchemaProvider provider;
+
+        public KeyspaceMetadataRef(KeyspaceMetadata initial, SchemaProvider provider)
+        {
+            this.initial = initial;
+            this.name = initial.name;
+            this.provider = provider;
+        }
+
+        public KeyspaceMetadata get()
+        {
+            if (initial != null)
+                return initial;
+            return provider.getKeyspaceMetadata(name);
+        }
+
+        public void unsetInitial()
+        {
+            this.initial = null;
+        }
     }
 }

src/java/org/apache/cassandra/db/MutationVerbHandler.java

@@ -24,7 +24,7 @@ import org.apache.cassandra.tracing.Tracing;
 
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
 
-public class MutationVerbHandler implements IVerbHandler<Mutation>
+public class MutationVerbHandler extends AbstractMutationVerbHandler<Mutation>
 {
     public static final MutationVerbHandler instance = new MutationVerbHandler();
 
@@ -51,7 +51,7 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
         InetAddressAndPort respondToAddress = message.respondTo();
         try
         {
-            message.payload.applyFuture().addCallback(o -> respond(message, respondToAddress), wto -> failed());
+            processMessage(message, respondToAddress);
         }
         catch (WriteTimeoutException wto)
         {
@@ -59,6 +59,11 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
         }
     }
 
+    protected void applyMutation(Message<Mutation> message, InetAddressAndPort respondToAddress)
+    {
+        message.payload.applyFuture().addCallback(o -> respond(message, respondToAddress), wto -> failed());
+    }
+
     private static void forwardToLocalNodes(Message<Mutation> originalMessage, ForwardingInfo forwardTo)
     {
         Message.Builder<Mutation> builder =

src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java

@@ -53,6 +53,7 @@ import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tracing.Tracing;
 
 /**
@@ -65,25 +66,27 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
     protected final DataRange dataRange;
     protected final Slices requestedSlices;
 
-    private PartitionRangeReadCommand(boolean isDigest,
+    @VisibleForTesting
-                                      int digestVersion,
+    protected PartitionRangeReadCommand(Epoch serializedAtEpoch,
-                                      boolean acceptsTransient,
+                                        boolean isDigest,
-                                      TableMetadata metadata,
+                                        int digestVersion,
-                                      long nowInSec,
+                                        boolean acceptsTransient,
-                                      ColumnFilter columnFilter,
+                                        TableMetadata metadata,
-                                      RowFilter rowFilter,
+                                        long nowInSec,
-                                      DataLimits limits,
+                                        ColumnFilter columnFilter,
-                                      DataRange dataRange,
+                                        RowFilter rowFilter,
-                                      Index.QueryPlan indexQueryPlan,
+                                        DataLimits limits,
-                                      boolean trackWarnings)
+                                        DataRange dataRange,
+                                        Index.QueryPlan indexQueryPlan,
+                                        boolean trackWarnings)
     {
-        super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings);
+        super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings);
         this.dataRange = dataRange;
         this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata());
-
     }
 
-    private static PartitionRangeReadCommand create(boolean isDigest,
+    private static PartitionRangeReadCommand create(Epoch serializedAtEpoch,
+                                                    boolean isDigest,
                                                     int digestVersion,
                                                     boolean acceptsTransient,
                                                     TableMetadata metadata,
@@ -109,7 +112,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                              indexQueryPlan,
                                                              trackWarnings);
         }
-        return new PartitionRangeReadCommand(isDigest,
+        return new PartitionRangeReadCommand(serializedAtEpoch,
+                                             isDigest,
                                              digestVersion,
                                              acceptsTransient,
                                              metadata,
@@ -129,7 +133,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                    DataLimits limits,
                                                    DataRange dataRange)
     {
-        return create(false,
+        return create(metadata.epoch,
+                      false,
                       0,
                       false,
                       metadata,
@@ -152,7 +157,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
      */
     public static PartitionRangeReadCommand allDataRead(TableMetadata metadata, long nowInSec)
     {
-        return create(false,
+        return create(metadata.epoch,
+                      false,
                       0,
                       false,
                       metadata,
@@ -202,7 +208,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
         // DataLimits.CQLGroupByLimits.GroupByAwareCounter assumes that if GroupingState.hasClustering(), then we're in
         // the middle of a group, but we can't make that assumption if we query and range "in advance" of where we are
         // on the ring.
-        return create(isDigestQuery(),
+        return create(serializedAtEpoch(),
+                      isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -217,7 +224,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
 
     public PartitionRangeReadCommand copy()
     {
-        return create(isDigestQuery(),
+        return create(serializedAtEpoch(),
+                      isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -233,7 +241,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
     @Override
     protected PartitionRangeReadCommand copyAsDigestQuery()
     {
-        return create(true,
+        return create(serializedAtEpoch(),
+                      true,
                       digestVersion(),
                       false,
                       metadata(),
@@ -249,7 +258,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
     @Override
     protected PartitionRangeReadCommand copyAsTransientQuery()
     {
-        return create(false,
+        return create(serializedAtEpoch(),
+                      false,
                       0,
                       true,
                       metadata(),
@@ -265,7 +275,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
     @Override
     public PartitionRangeReadCommand withUpdatedLimit(DataLimits newLimits)
     {
-        return create(isDigestQuery(),
+        return create(serializedAtEpoch(),
+                      isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -281,7 +292,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
     @Override
     public PartitionRangeReadCommand withUpdatedLimitsAndDataRange(DataLimits newLimits, DataRange newDataRange)
     {
-        return create(isDigestQuery(),
+        return create(serializedAtEpoch(),
+                      isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -515,6 +527,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
     {
         public ReadCommand deserialize(DataInputPlus in,
                                        int version,
+                                       Epoch serializedAtEpoch,
                                        boolean isDigest,
                                        int digestVersion,
                                        boolean acceptsTransient,
@@ -527,7 +540,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
         throws IOException
         {
             DataRange range = DataRange.serializer.deserialize(in, version, metadata);
-            return PartitionRangeReadCommand.create(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
+            return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
         }
     }
 
@@ -545,7 +558,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                       Index.QueryPlan indexQueryPlan,
                                                       boolean trackWarnings)
         {
-            super(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
+            super(metadata.epoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
         }
 
         @Override

src/java/org/apache/cassandra/db/ReadCommand.java

@@ -38,7 +38,10 @@ import org.slf4j.LoggerFactory;
 import io.netty.util.concurrent.FastThreadLocal;
 import org.apache.cassandra.config.*;
 import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.exceptions.CoordinatorBehindException;
 import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.exceptions.UnknownTableException;
+import org.apache.cassandra.metrics.TCMMetrics;
 import org.apache.cassandra.net.MessageFlag;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.ParamType;
@@ -67,9 +70,12 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.CassandraUInt;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.TimeUUID;
 
@@ -100,6 +106,7 @@ public abstract class ReadCommand extends AbstractReadQuery
 
     private final boolean isDigestQuery;
     private final boolean acceptsTransient;
+    private final Epoch serializedAtEpoch;
     // if a digest query, the version for which the digest is expected. Ignored if not a digest.
     private int digestVersion;
 
@@ -112,6 +119,7 @@ public abstract class ReadCommand extends AbstractReadQuery
     {
         public abstract ReadCommand deserialize(DataInputPlus in,
                                                 int version,
+                                                Epoch serializedAtEpoch,
                                                 boolean isDigest,
                                                 int digestVersion,
                                                 boolean acceptsTransient,
@@ -136,7 +144,8 @@ public abstract class ReadCommand extends AbstractReadQuery
         }
     }
 
-    protected ReadCommand(Kind kind,
+    protected ReadCommand(Epoch serializedAtEpoch,
+                          Kind kind,
                           boolean isDigestQuery,
                           int digestVersion,
                           boolean acceptsTransient,
@@ -158,6 +167,7 @@ public abstract class ReadCommand extends AbstractReadQuery
         this.acceptsTransient = acceptsTransient;
         this.indexQueryPlan = indexQueryPlan;
         this.trackWarnings = trackWarnings;
+        this.serializedAtEpoch = serializedAtEpoch;
     }
 
     public static ReadCommand getCommand()
@@ -197,6 +207,15 @@ public abstract class ReadCommand extends AbstractReadQuery
         return isDigestQuery;
     }
 
+    /**
+     * the schema version on the table when serializing this read command
+     * @return
+     */
+    public Epoch serializedAtEpoch()
+    {
+        return serializedAtEpoch;
+    }
+
     /**
      * If the query is a digest one, the requested digest version.
      *
@@ -1011,6 +1030,11 @@ public abstract class ReadCommand extends AbstractReadQuery
     @VisibleForTesting
     public static class Serializer implements IVersionedSerializer<ReadCommand>
     {
+        private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 10L, TimeUnit.SECONDS);
+        private static final NoSpamLogger.NoSpamLogStatement schemaMismatchStmt =
+            noSpamLogger.getStatement("Schema epoch mismatch during read command deserialization. " +
+                                      "TableId: {}, remote epoch: {}, local epoch: {}", 10L, TimeUnit.SECONDS);
+
         private final SchemaProvider schema;
 
         public Serializer()
@@ -1075,6 +1099,8 @@ public abstract class ReadCommand extends AbstractReadQuery
             if (command.isDigestQuery())
                 out.writeUnsignedVInt32(command.digestVersion());
             command.metadata().id.serialize(out);
+            if (version >= MessagingService.VERSION_50)
+                Epoch.serializer.serialize(command.serializedAtEpoch, out);
             out.writeInt(version >= MessagingService.VERSION_50 ? CassandraUInt.fromLong(command.nowInSec()) : (int) command.nowInSec());
             ColumnFilter.serializer.serialize(command.columnFilter(), out, version);
             RowFilter.serializer.serialize(command.rowFilter(), out, version);
@@ -1098,28 +1124,47 @@ public abstract class ReadCommand extends AbstractReadQuery
             // better complain loudly than doing the wrong thing.
             if (isForThrift(flags))
                 throw new IllegalStateException("Received a command with the thrift flag set. "
-                                              + "This means thrift is in use in a mixed 3.0/3.X and 4.0+ cluster, "
+                                                + "This means thrift is in use in a mixed 3.0/3.X and 4.0+ cluster, "
-                                              + "which is unsupported. Make sure to stop using thrift before "
+                                                + "which is unsupported. Make sure to stop using thrift before "
-                                              + "upgrading to 4.0");
+                                                + "upgrading to 4.0");
 
             boolean hasIndex = hasIndex(flags);
-            int digestVersion = isDigest ? in.readUnsignedVInt32() : 0;
+            int digestVersion = isDigest ? (int)in.readUnsignedVInt() : 0;
-            TableMetadata metadata = schema.getExistingTableMetadata(TableId.deserialize(in));
+            TableId tableId = TableId.deserialize(in);
-            long nowInSec = version >= MessagingService.VERSION_50 ? CassandraUInt.toLong(in.readInt()) : in.readInt();
-            ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata);
-            RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata);
-            DataLimits limits = DataLimits.serializer.deserialize(in, version,  metadata);
 
+            Epoch schemaVersion = null;
+            if (version >= MessagingService.VERSION_50)
+                schemaVersion = Epoch.serializer.deserialize(in);
+            TableMetadata tableMetadata;
+            try
+            {
+                tableMetadata = schema.getExistingTableMetadata(tableId);
+            }
+            catch (UnknownTableException e)
+            {
+                ClusterMetadata metadata = ClusterMetadata.current();
+                Epoch localCurrentEpoch = metadata.epoch;
+                if (schemaVersion != null && localCurrentEpoch.isAfter(schemaVersion))
+                {
+                    TCMMetrics.instance.coordinatorBehindSchema.mark();
+                    throw new CoordinatorBehindException(e.getMessage());
+                }
+                throw e;
+            }
+            long nowInSec = version >= MessagingService.VERSION_50 ? CassandraUInt.toLong(in.readInt()) : in.readInt();
+            ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, tableMetadata);
+            RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, tableMetadata);
+            DataLimits limits = DataLimits.serializer.deserialize(in, version,  tableMetadata);
             Index.QueryPlan indexQueryPlan = null;
             if (hasIndex)
             {
-                IndexMetadata index = deserializeIndexMetadata(in, version, metadata);
+                IndexMetadata index = deserializeIndexMetadata(in, version, tableMetadata);
-                Index.Group indexGroup =  Keyspace.openAndGetStore(metadata).indexManager.getIndexGroup(index);
+                Index.Group indexGroup =  Keyspace.openAndGetStore(tableMetadata).indexManager.getIndexGroup(index);
                 if (indexGroup != null)
                     indexQueryPlan = indexGroup.queryPlanFor(rowFilter);
             }
 
-            return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
+            return kind.selectionDeserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, acceptsTransient, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
         }
 
         private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException
@@ -1144,6 +1189,7 @@ public abstract class ReadCommand extends AbstractReadQuery
             return 2 // kind + flags
                    + (command.isDigestQuery() ? TypeSizes.sizeofUnsignedVInt(command.digestVersion()) : 0)
                    + command.metadata().id.serializedSize()
+                   + (version >= MessagingService.VERSION_50 ? Epoch.serializer.serializedSize(command.metadata().epoch) : 0)
                    + TypeSizes.INT_SIZE // command.nowInSec() is serialized as uint
                    + ColumnFilter.serializer.serializedSize(command.columnFilter(), version)
                    + RowFilter.serializer.serializedSize(command.rowFilter(), version)

src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java

@@ -22,15 +22,24 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.exceptions.CoordinatorBehindException;
+import org.apache.cassandra.exceptions.InvalidRoutingException;
+import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.exceptions.QueryCancelledException;
 import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.metrics.TCMMetrics;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 
@@ -42,16 +51,17 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
 
     public void doVerb(Message<ReadCommand> message)
     {
-        if (StorageService.instance.isBootstrapMode())
+        if (message.epoch().isAfter(Epoch.EMPTY))
         {
-            throw new RuntimeException("Cannot service reads while bootstrapping!");
+            ClusterMetadata metadata = ClusterMetadata.current();
+            metadata = checkTokenOwnership(metadata, message);
+            metadata = checkSchemaVersion(metadata, message);
         }
 
-        ReadCommand command = message.payload;
-        validateTransientStatus(message);
         MessageParams.reset();
 
         long timeout = message.expiresAtNanos() - message.createdAtNanos();
+        ReadCommand command = message.payload;
         command.setMonitoringTime(message.createdAtNanos(), message.isCrossNode(), timeout, DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS));
 
         if (message.trackWarnings())
@@ -103,40 +113,110 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
         }
     }
 
-    private void validateTransientStatus(Message<ReadCommand> message)
+    private ClusterMetadata checkSchemaVersion(ClusterMetadata metadata, Message<ReadCommand> message)
+    {
+        ReadCommand readCommand = message.payload;
+
+        if (SchemaConstants.isSystemKeyspace(readCommand.metadata().keyspace) ||
+            readCommand.serializedAtEpoch() == null) // don't try to catch up with pre-5.0 nodes
+            return metadata;
+
+        Keyspace ks = metadata.schema.getKeyspace(readCommand.metadata().keyspace);
+        ColumnFamilyStore cfs = ks != null ? ks.getColumnFamilyStore(readCommand.metadata().id) : null;
+        Epoch localComparisonEpoch = metadata.epoch;
+        if (cfs != null)
+            localComparisonEpoch = cfs.metadata().epoch;
+
+        if (localComparisonEpoch.isBefore(readCommand.serializedAtEpoch()))
+            metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch());
+        else if (localComparisonEpoch.isAfter(readCommand.serializedAtEpoch()))
+        {
+            TCMMetrics.instance.coordinatorBehindSchema.mark();
+            throw new CoordinatorBehindException(String.format("Coordinator schema for %s.%s with epoch %s is behind our schema %s",
+                                                               message.payload.metadata().keyspace,
+                                                               message.payload.metadata().name,
+                                                               readCommand.serializedAtEpoch(),
+                                                               localComparisonEpoch));
+        }
+        ks = metadata.schema.getKeyspace(readCommand.metadata().keyspace);
+        if (ks == null || ks.getColumnFamilyStore(readCommand.metadata().id) == null)
+            throw new IllegalStateException("Unknown table " + readCommand.metadata().id +" after fetching remote log entries");
+        return metadata;
+    }
+
+    private ClusterMetadata checkTokenOwnership(ClusterMetadata metadata, Message<ReadCommand> message)
     {
         ReadCommand command = message.payload;
         if (command.metadata().isVirtual())
-            return;
+            return metadata;
-        Token token;
+
+        if (command.isTopK())
+            return metadata;
 
         if (command instanceof SinglePartitionReadCommand)
-            token = ((SinglePartitionReadCommand) command).partitionKey().getToken();
+        {
+            Token token = ((SinglePartitionReadCommand) command).partitionKey().getToken();
+            Replica localReplica = getLocalReplica(metadata, token, command.metadata().keyspace);
+            if (localReplica == null)
+            {
+                metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch());
+                localReplica = getLocalReplica(metadata, token, command.metadata().keyspace);
+            }
+            if (localReplica == null)
+            {
+                StorageService.instance.incOutOfRangeOperationCount();
+                Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc();
+                throw InvalidRoutingException.forTokenRead(message.from(), token, metadata.epoch, message.payload);
+            }
+
+            if (!command.acceptsTransient() && localReplica.isTransient())
+            {
+                MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+                throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s",
+                                                                command.acceptsTransient() ? "transient" : "full",
+                                                                localReplica.isTransient() ? "transient" : "full",
+                                                                this));
+            }
+        }
         else
-            token = ((PartitionRangeReadCommand) command).dataRange().keyRange().right.getToken();
-
-        Replica replica = Keyspace.open(command.metadata().keyspace)
-                                  .getReplicationStrategy()
-                                  .getLocalReplicaFor(token);
-
-        if (replica == null)
         {
-            if (command.isTopK())
+            AbstractBounds<PartitionPosition> range = ((PartitionRangeReadCommand) command).dataRange().keyRange();
-                return;
 
-            logger.warn("Received a read request from {} for a range that is not owned by the current replica {}.",
+            // TODO: preexisting issue: for the range queries or queries that span multiple replicas, we can only make requests where the right token is owned, but not the left one
-                        message.from(),
+            Replica maxTokenLocalReplica = getLocalReplica(metadata, range.right.getToken(), command.metadata().keyspace);
-                        command);
+            if (maxTokenLocalReplica == null)
-            return;
+            {
-        }
+                metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch());
+                maxTokenLocalReplica = getLocalReplica(metadata, range.right.getToken(), command.metadata().keyspace);
+            }
+            if (maxTokenLocalReplica == null)
+            {
+                StorageService.instance.incOutOfRangeOperationCount();
+                Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc();
+                throw InvalidRoutingException.forRangeRead(message.from(), range, metadata.epoch, message.payload);
+            }
 
-        if (!command.acceptsTransient() && replica.isTransient())
+
-        {
+            // TODO: preexisting issue: we should change the whole range for transient-ness, not just the right token
-            MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+            if (command.acceptsTransient() != maxTokenLocalReplica.isTransient())
-            throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s",
+            {
-                                                            command.acceptsTransient() ? "transient" : "full",
+                MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
-                                                            replica.isTransient() ? "transient" : "full",
+                throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s",
-                                                            this));
+                                                                command.acceptsTransient() ? "transient" : "full",
+                                                                maxTokenLocalReplica.isTransient() ? "transient" : "full",
+                                                                this));
+            }
         }
+        return metadata;
+    }
+
+    private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace)
+    {
+        return metadata.placements
+               .get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication)
+               .reads
+               .forToken(token)
+               .get()
+               .lookup(FBUtilities.getBroadcastAddressAndPort());
     }
 }

src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java

@@ -17,17 +17,17 @@
  */
 package org.apache.cassandra.db;
 
-import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 
-public class ReadRepairVerbHandler implements IVerbHandler<Mutation>
+public class ReadRepairVerbHandler extends AbstractMutationVerbHandler<Mutation>
 {
     public static final ReadRepairVerbHandler instance = new ReadRepairVerbHandler();
 
-    public void doVerb(Message<Mutation> message)
+    void applyMutation(Message<Mutation> message, InetAddressAndPort respondToAddress)
     {
         message.payload.apply();
-        MessagingService.instance().send(message.emptyResponse(), message.from());
+        MessagingService.instance().send(message.emptyResponse(), respondToAddress);
     }
 }

src/java/org/apache/cassandra/db/ReadResponse.java

@@ -34,6 +34,8 @@ import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
+import static org.apache.cassandra.db.RepairedDataInfo.NO_OP_REPAIRED_DATA_INFO;
+
 public abstract class ReadResponse
 {
     // Serializer for single partition read response
@@ -48,6 +50,11 @@ public abstract class ReadResponse
         return new LocalDataResponse(data, command, rdi);
     }
 
+    public static ReadResponse createDataResponse(UnfilteredPartitionIterator data, ReadCommand command)
+    {
+        return new LocalDataResponse(data, command, NO_OP_REPAIRED_DATA_INFO);
+    }
+
     public static ReadResponse createSimpleDataResponse(UnfilteredPartitionIterator data, ColumnFilter selection)
     {
         return new LocalDataResponse(data, selection);

src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java

@@ -76,6 +76,7 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.btree.BTreeSet;
@@ -91,7 +92,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     protected final ClusteringIndexFilter clusteringIndexFilter;
 
     @VisibleForTesting
-    protected SinglePartitionReadCommand(boolean isDigest,
+    protected SinglePartitionReadCommand(Epoch serializedAtEpoch,
+                                         boolean isDigest,
                                          int digestVersion,
                                          boolean acceptsTransient,
                                          TableMetadata metadata,
@@ -104,24 +106,25 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                          Index.QueryPlan indexQueryPlan,
                                          boolean trackWarnings)
     {
-        super(Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings);
+        super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings);
         assert partitionKey.getPartitioner() == metadata.partitioner;
         this.partitionKey = partitionKey;
         this.clusteringIndexFilter = clusteringIndexFilter;
     }
 
-    private static SinglePartitionReadCommand create(boolean isDigest,
+    private static SinglePartitionReadCommand create(Epoch serializedAtEpoch,
-                                                    int digestVersion,
+                                                     boolean isDigest,
-                                                    boolean acceptsTransient,
+                                                     int digestVersion,
-                                                    TableMetadata metadata,
+                                                     boolean acceptsTransient,
-                                                    long nowInSec,
+                                                     TableMetadata metadata,
-                                                    ColumnFilter columnFilter,
+                                                     long nowInSec,
-                                                    RowFilter rowFilter,
+                                                     ColumnFilter columnFilter,
-                                                    DataLimits limits,
+                                                     RowFilter rowFilter,
-                                                    DecoratedKey partitionKey,
+                                                     DataLimits limits,
-                                                    ClusteringIndexFilter clusteringIndexFilter,
+                                                     DecoratedKey partitionKey,
-                                                    Index.QueryPlan indexQueryPlan,
+                                                     ClusteringIndexFilter clusteringIndexFilter,
-                                                    boolean trackWarnings)
+                                                     Index.QueryPlan indexQueryPlan,
+                                                     boolean trackWarnings)
     {
         if (metadata.isVirtual())
         {
@@ -138,7 +141,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                               indexQueryPlan,
                                                               trackWarnings);
         }
-        return new SinglePartitionReadCommand(isDigest,
+        return new SinglePartitionReadCommand(serializedAtEpoch,
+                                              isDigest,
                                               digestVersion,
                                               acceptsTransient,
                                               metadata,
@@ -175,7 +179,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                     ClusteringIndexFilter clusteringIndexFilter,
                                                     Index.QueryPlan indexQueryPlan)
     {
-        return create(false,
+        return create(metadata.epoch,
+                      false,
                       0,
                       false,
                       metadata,
@@ -352,7 +357,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
 
     public SinglePartitionReadCommand copy()
     {
-        return create(isDigestQuery(),
+        return create(serializedAtEpoch(),
+                      isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -369,7 +375,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     @Override
     protected SinglePartitionReadCommand copyAsDigestQuery()
     {
-        return create(true,
+        return create(serializedAtEpoch(),
+                      true,
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -386,7 +393,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     @Override
     protected SinglePartitionReadCommand copyAsTransientQuery()
     {
-        return create(false,
+        return create(serializedAtEpoch(),
+                      false,
                       0,
                       true,
                       metadata(),
@@ -403,7 +411,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     @Override
     public SinglePartitionReadCommand withUpdatedLimit(DataLimits newLimits)
     {
-        return create(isDigestQuery(),
+        return create(serializedAtEpoch(),
+                      isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
                       metadata(),
@@ -1302,6 +1311,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     {
         public ReadCommand deserialize(DataInputPlus in,
                                        int version,
+                                       Epoch serializedAtEpoch,
                                        boolean isDigest,
                                        int digestVersion,
                                        boolean acceptsTransient,
@@ -1315,7 +1325,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         {
             DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readBuffer(in, DatabaseDescriptor.getMaxValueSize()));
             ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
-            return SinglePartitionReadCommand.create(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
+            return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
         }
     }
 
@@ -1362,7 +1372,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                          Index.QueryPlan indexQueryPlan,
                                                          boolean trackWarnings)
         {
-            super(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter, indexQueryPlan, trackWarnings);
+            super(metadata.epoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter, indexQueryPlan, trackWarnings);
         }
 
         @Override

src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java

@@ -19,7 +19,10 @@ package org.apache.cassandra.db;
 
 import java.util.*;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,15 +32,17 @@ import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaChangeListener;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Refs;
 
+import static org.apache.cassandra.tcm.compatibility.TokenRingUtils.getAllRanges;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
@@ -62,8 +67,7 @@ public class SizeEstimatesRecorder implements SchemaChangeListener, Runnable
 
     public void run()
     {
-        TokenMetadata metadata = StorageService.instance.getTokenMetadata().cloneOnlyTokenMap();
+        if (!ClusterMetadata.current().directory.allAddresses().contains(FBUtilities.getBroadcastAddressAndPort()))
-        if (!metadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
         {
             logger.debug("Node is not part of the ring; not recording size estimates");
             return;
@@ -73,6 +77,9 @@ public class SizeEstimatesRecorder implements SchemaChangeListener, Runnable
 
         for (Keyspace keyspace : Keyspace.nonLocalStrategy())
         {
+            if (keyspace.getMetadata().params.replication.isMeta())
+                continue;
+
             // In tools the call to describe_splits_ex() used to be coupled with the call to describe_local_ring() so
             // most access was for the local primary range; after creating the size_estimates table this was changed
             // to be the primary range.
@@ -88,7 +95,7 @@ public class SizeEstimatesRecorder implements SchemaChangeListener, Runnable
             // range.  If we publish multiple ranges downstream integrations may start to see duplicate data.
             // See CASSANDRA-15637
             Collection<Range<Token>> primaryRanges = StorageService.instance.getPrimaryRanges(keyspace.getName());
-            Collection<Range<Token>> localPrimaryRanges = StorageService.instance.getLocalPrimaryRange();
+            Collection<Range<Token>> localPrimaryRanges = getLocalPrimaryRange();
             boolean rangesAreEqual = primaryRanges.equals(localPrimaryRanges);
             for (ColumnFamilyStore table : keyspace.getColumnFamilyStores())
             {
@@ -116,6 +123,34 @@ public class SizeEstimatesRecorder implements SchemaChangeListener, Runnable
         }
     }
 
+    @VisibleForTesting
+    public static Collection<Range<Token>> getLocalPrimaryRange()
+    {
+        ClusterMetadata metadata = ClusterMetadata.current();
+        NodeId localNodeId = metadata.myNodeId();
+        return getLocalPrimaryRange(metadata, localNodeId);
+    }
+
+    @VisibleForTesting
+    public static Collection<Range<Token>> getLocalPrimaryRange(ClusterMetadata metadata, NodeId nodeId)
+    {
+        String dc = metadata.directory.location(nodeId).datacenter;
+        Set<Token> tokens = new HashSet<>(metadata.tokenMap.tokens(nodeId));
+
+        // filter tokens to the single DC
+        List<Token> filteredTokens = Lists.newArrayList();
+        for (Token token : metadata.tokenMap.tokens())
+        {
+            NodeId owner = metadata.tokenMap.owner(token);
+            if (dc.equals(metadata.directory.location(owner).datacenter))
+                filteredTokens.add(token);
+        }
+        return getAllRanges(filteredTokens).stream()
+                                           .filter(t -> tokens.contains(t.right))
+                                           .collect(Collectors.toList());
+    }
+
+    @SuppressWarnings("resource")
     private static Map<Range<Token>, Pair<Long, Long>> computeSizeEstimates(ColumnFamilyStore table, Collection<Range<Token>> ranges)
     {
         // for each local primary range, estimate (crudely) mean partition size and partitions count.

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -36,7 +36,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 import javax.management.openmbean.OpenDataException;
@@ -59,7 +58,6 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
-import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.compaction.CompactionHistoryTabularData;
 import org.apache.cassandra.db.marshal.ByteBufferAccessor;
@@ -77,6 +75,9 @@ import org.apache.cassandra.dht.LocalPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.HeartBeatState;
+import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.io.sstable.SSTableId;
 import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
 import org.apache.cassandra.io.util.DataInputBuffer;
@@ -110,6 +111,10 @@ import org.apache.cassandra.service.paxos.PaxosState;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedIndex;
 import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.Sealed;
+import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.CassandraVersion;
@@ -127,6 +132,14 @@ import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithNowInSec;
 import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
+import static org.apache.cassandra.gms.ApplicationState.DC;
+import static org.apache.cassandra.gms.ApplicationState.HOST_ID;
+import static org.apache.cassandra.gms.ApplicationState.INTERNAL_ADDRESS_AND_PORT;
+import static org.apache.cassandra.gms.ApplicationState.NATIVE_ADDRESS_AND_PORT;
+import static org.apache.cassandra.gms.ApplicationState.RACK;
+import static org.apache.cassandra.gms.ApplicationState.RELEASE_VERSION;
+import static org.apache.cassandra.gms.ApplicationState.STATUS_WITH_PORT;
+import static org.apache.cassandra.gms.ApplicationState.TOKENS;
 import static org.apache.cassandra.service.paxos.Commit.latest;
 import static org.apache.cassandra.utils.CassandraVersion.NULL_VERSION;
 import static org.apache.cassandra.utils.CassandraVersion.UNREADABLE_VERSION;
@@ -163,6 +176,10 @@ public final class SystemKeyspace
     public static final String PREPARED_STATEMENTS = "prepared_statements";
     public static final String REPAIRS = "repairs";
     public static final String TOP_PARTITIONS = "top_partitions";
+    public static final String METADATA_LOG = "local_metadata_log";
+    public static final String SNAPSHOT_TABLE_NAME = "metadata_snapshots";
+    public static final String SEALED_PERIODS_TABLE_NAME = "metadata_sealed_periods";
+    public static final String LAST_SEALED_PERIOD_TABLE_NAME = "metadata_last_sealed_period";
 
     /**
      * By default the system keyspace tables should be stored in a single data directory to allow the server
@@ -195,13 +212,15 @@ public final class SystemKeyspace
         COMPACTION_HISTORY, SSTABLE_ACTIVITY_V2, TABLE_ESTIMATES, TABLE_ESTIMATES_TYPE_PRIMARY,
         TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY, AVAILABLE_RANGES_V2, TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS,
         BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS, LEGACY_PEERS, LEGACY_PEER_EVENTS,
-        LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY);
+        LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY,
+        METADATA_LOG, SNAPSHOT_TABLE_NAME, SEALED_PERIODS_TABLE_NAME, LAST_SEALED_PERIOD_TABLE_NAME);
 
     public static final Set<String> TABLE_NAMES = ImmutableSet.of(
         BATCHES, PAXOS, PAXOS_REPAIR_HISTORY, BUILT_INDEXES, LOCAL, PEERS_V2, PEER_EVENTS_V2, 
         COMPACTION_HISTORY, SSTABLE_ACTIVITY_V2, TABLE_ESTIMATES, AVAILABLE_RANGES_V2, TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS, 
         BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS, LEGACY_PEERS, LEGACY_PEER_EVENTS, 
-        LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY);
+        LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY,
+        METADATA_LOG, SNAPSHOT_TABLE_NAME, SEALED_PERIODS_TABLE_NAME, LAST_SEALED_PERIOD_TABLE_NAME);
 
     public static final TableMetadata Batches =
         parse(BATCHES,
@@ -466,7 +485,45 @@ public final class SystemKeyspace
           + "cfids set<uuid>, "
           + "PRIMARY KEY (parent_id))").build();
 
-    /** @deprecated See CASSANDRA-7544 */
+    public static final TableMetadata LocalMetadataLog =
+        parse(METADATA_LOG,
+              "Local Metadata Log",
+              "CREATE TABLE %s ("
+              + "period bigint,"
+              + "current_epoch bigint static,"
+              + "epoch bigint,"
+              + "entry_id bigint,"
+              + "transformation blob,"
+              + "kind text,"
+              + "PRIMARY KEY (period, epoch))")
+        .compaction(CompactionParams.twcs(ImmutableMap.of("compaction_window_unit","DAYS",
+                                                          "compaction_window_size","1")))
+        .build();
+
+    public static final TableMetadata Snapshots = parse(SNAPSHOT_TABLE_NAME,
+                                                        "ClusterMetadata snapshots",
+                                                        "CREATE TABLE IF NOT EXISTS %s (" +
+                                                        "epoch bigint PRIMARY KEY," +
+                                                        "period bigint," +
+                                                        "snapshot blob)")
+                                                  .build();
+
+    public static final TableMetadata SealedPeriods = parse(SEALED_PERIODS_TABLE_NAME,
+                                                            "ClusterMetadata sealed periods",
+                                                            "CREATE TABLE IF NOT EXISTS %s (" +
+                                                            "max_epoch bigint PRIMARY KEY," +
+                                                            "period bigint)")
+                                                      .partitioner(new LocalPartitioner(LongType.instance))
+                                                      .build();
+
+    public static final TableMetadata LastSealedPeriod = parse(LAST_SEALED_PERIOD_TABLE_NAME,
+                                                               "ClusterMetadata last sealed period",
+                                                               "CREATE TABLE IF NOT EXISTS %s (" +
+                                                               "key text PRIMARY KEY," +
+                                                               "epoch bigint," +
+                                                               "period bigint)")
+                                                         .build();
+
     @Deprecated(since = "4.0")
     private static final TableMetadata LegacyPeers =
         parse(LEGACY_PEERS,
@@ -557,7 +614,11 @@ public final class SystemKeyspace
                          BuiltViews,
                          PreparedStatements,
                          Repairs,
-                         TopPartitions);
+                         TopPartitions,
+                         LocalMetadataLog,
+                         LastSealedPeriod,
+                         SealedPeriods,
+                         Snapshots);
     }
 
     private static volatile Map<TableId, Pair<CommitLogPosition, Long>> truncationRecords;
@@ -567,16 +628,31 @@ public final class SystemKeyspace
         NEEDS_BOOTSTRAP,
         COMPLETED,
         IN_PROGRESS,
-        DECOMMISSIONED
+        DECOMMISSIONED;
+
+        public static BootstrapState fromNodeState(NodeState nodeState)
+        {
+            if (nodeState == null) // todo, handle this properly
+                return DECOMMISSIONED;
+            switch (nodeState)
+            {
+                case REGISTERED:
+                    return NEEDS_BOOTSTRAP;
+                case BOOTSTRAPPING:
+                case BOOT_REPLACING:
+                    return IN_PROGRESS;
+                case JOINED:
+                case LEAVING:
+                case MOVING:
+                    return COMPLETED;
+                case LEFT:
+                default:
+                    return DECOMMISSIONED;
+            }
+        }
     }
 
     public static void persistLocalMetadata()
-    {
-        persistLocalMetadata(UUID::randomUUID);
-    }
-
-    @VisibleForTesting
-    public static void persistLocalMetadata(Supplier<UUID> nodeIdSupplier)
     {
         String req = "INSERT INTO system.%s (" +
                      "key," +
@@ -610,13 +686,6 @@ public final class SystemKeyspace
                             DatabaseDescriptor.getStoragePort(),
                             FBUtilities.getJustLocalAddress(),
                             DatabaseDescriptor.getStoragePort());
-
-        // We should store host ID as soon as possible in the system.local table and flush that table to disk so that
-        // we can be sure that those changes are stored in sstable and not in the commit log (see CASSANDRA-18153).
-        // It is very unlikely that when upgrading the host id is not flushed to disk, but if that's the case, we limit
-        // this change only to the new installations or the user should just flush system.local table.
-        if (!CommitLog.instance.hasFilesToReplay())
-            SystemKeyspace.getOrInitializeLocalHostId(nodeIdSupplier);
     }
 
     public static void updateCompactionHistory(TimeUUID taskId,
@@ -831,7 +900,10 @@ public final class SystemKeyspace
     public static synchronized void updateTokens(InetAddressAndPort ep, Collection<Token> tokens)
     {
         if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
+        {
+            updateLocalTokens(tokens);
             return;
+        }
 
         String req = "INSERT INTO system.%s (peer, tokens) VALUES (?, ?)";
         executeInternal(String.format(req, LEGACY_PEERS), ep.getAddress(), tokensAsSet(tokens));
@@ -895,11 +967,11 @@ public final class SystemKeyspace
         executeInternal(format(req, LOCAL, LOCAL), version);
     }
 
-    private static Set<String> tokensAsSet(Collection<Token> tokens)
+    public static Set<String> tokensAsSet(Collection<Token> tokens)
     {
         if (tokens.isEmpty())
             return Collections.emptySet();
-        Token.TokenFactory factory = StorageService.instance.getTokenFactory();
+        Token.TokenFactory factory = ClusterMetadata.current().partitioner.getTokenFactory();
         Set<String> s = new HashSet<>(tokens.size());
         for (Token tk : tokens)
             s.add(factory.toString(tk));
@@ -908,7 +980,7 @@ public final class SystemKeyspace
 
     private static Collection<Token> deserializeTokens(Collection<String> tokensStrings)
     {
-        Token.TokenFactory factory = StorageService.instance.getTokenFactory();
+        Token.TokenFactory factory = ClusterMetadata.current().partitioner.getTokenFactory();
         List<Token> tokens = new ArrayList<>(tokensStrings.size());
         for (String tk : tokensStrings)
             tokens.add(factory.fromString(tk));
@@ -928,9 +1000,10 @@ public final class SystemKeyspace
     }
 
     /**
+     *
      * This method is used to update the System Keyspace with the new tokens for this node
      */
-    public static synchronized void updateTokens(Collection<Token> tokens)
+    public static synchronized void updateLocalTokens(Collection<Token> tokens)
     {
         assert !tokens.isEmpty() : "removeEndpoint should be used instead";
 
@@ -1240,27 +1313,6 @@ public final class SystemKeyspace
         return null;
     }
 
-    /**
-     * Read the host ID from the system keyspace, creating (and storing) one if
-     * none exists.
-     */
-    public static synchronized UUID getOrInitializeLocalHostId()
-    {
-        return getOrInitializeLocalHostId(UUID::randomUUID);
-    }
-
-    private static synchronized UUID getOrInitializeLocalHostId(Supplier<UUID> nodeIdSupplier)
-    {
-        UUID hostId = getLocalHostId();
-        if (hostId != null)
-            return hostId;
-
-        // ID not found, generate a new one, persist, and then return it.
-        hostId = nodeIdSupplier.get();
-        logger.warn("No host ID found, created {} (Note: This should happen exactly once per node).", hostId);
-        return setLocalHostId(hostId);
-    }
-
     /**
      * Sets the local host ID explicitly.  Should only be called outside of SystemTable when replacing a node.
      */
@@ -1316,6 +1368,20 @@ public final class SystemKeyspace
         return null;
     }
 
+    public static Set<String> allKnownDatacenters()
+    {
+        Set<String> dcs = new HashSet<>();
+        dcs.add(getDatacenter());
+        String req = "SELECT data_center FROM system.%s";
+        UntypedResultSet result = executeInternal(format(req, PEERS_V2));
+        if (result != null)
+        {
+            for (UntypedResultSet.Row row : result)
+                dcs.add(row.getString("data_center"));
+        }
+        return dcs;
+    }
+
     /**
      * Load the current paxos state for the table and key
      */
@@ -1937,4 +2003,122 @@ public final class SystemKeyspace
             return TopPartitionTracker.StoredTopPartitions.EMPTY;
         }
     }
+
+    public static void storeSnapshot(Epoch epoch, long period, ByteBuffer snapshot)
+    {
+        logger.info("Storing snapshot of cluster metadata at epoch {} (period {})", epoch, period);
+        String query = String.format("INSERT INTO %s.%s (epoch, period, snapshot) VALUES (?, ?, ?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME);
+        executeInternal(query, epoch.getEpoch(), period, snapshot);
+    }
+
+    public static ByteBuffer getSnapshot(Epoch epoch)
+    {
+        logger.info("Getting snapshot of epoch = {}", epoch);
+        String query = String.format("SELECT SNAPSHOT FROM %s.%s WHERE epoch = ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME);
+        UntypedResultSet res = executeInternal(query, epoch.getEpoch());
+        if (res == null || res.isEmpty())
+            return null;
+        return res.one().getBytes("snapshot").duplicate();
+    }
+
+    public static Sealed findSealedPeriodForEpochScan(Epoch search)
+    {
+        String query = String.format("SELECT max_epoch, period FROM %s.%s WHERE max_epoch >= ? LIMIT 1 ALLOW FILTERING", SchemaConstants.SYSTEM_KEYSPACE_NAME, SEALED_PERIODS_TABLE_NAME);
+        UntypedResultSet res = executeInternal(query, search.getEpoch());
+        if (res != null && !res.isEmpty())
+        {
+            long period = res.one().getLong("period");
+            long epoch = res.one().getLong("max_epoch");
+            return new Sealed(period, epoch);
+        }
+
+        // nothing found for this epoch, is the table empty or is the search epoch > the maximum
+        query = String.format("SELECT max_epoch, period FROM %s.%s LIMIT 1", SchemaConstants.SYSTEM_KEYSPACE_NAME, SEALED_PERIODS_TABLE_NAME);
+        res = executeInternal(query);
+        // table is empty, so any scan for the epoch will have to begin at Period.EMPTY
+        if (res == null || res.isEmpty())
+            return Sealed.EMPTY;
+
+        // the index table has some data, but is the search target greater than the max epoch in last sealed period?
+        // This query is relatively costly, so we do it last. Retain the min period/epoch that we did find in the
+        // previous query just in case we need them
+        // TODO add a nodetool command to rebuild the local sealed periods table
+        long lowestPeriod = res.one().getLong("period");
+        long lowestMaxEpoch = res.one().getLong("max_epoch");
+        logger.info("Scanning sealed periods by epoch table, this may be an expensive operation and the index table {} should be rebuilt", SEALED_PERIODS_TABLE_NAME);
+        query = String.format("SELECT max(max_epoch) AS max_epoch FROM %s.%s LIMIT 1 ALLOW FILTERING;", SchemaConstants.SYSTEM_KEYSPACE_NAME, SEALED_PERIODS_TABLE_NAME);
+        res = executeInternal(query);
+
+        // should never happen because the previous query returned the min, but just in case the table has been
+        // truncated since then, return the min Sealed.
+        if (res == null || res.isEmpty())
+            return new Sealed(lowestPeriod, lowestMaxEpoch);
+
+        // use the max epoch to look up the sealed period
+        long maxEpoch = res.one().getLong("max_epoch");
+        query = String.format("SELECT period FROM %s.%s WHERE max_epoch = ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, SEALED_PERIODS_TABLE_NAME);
+        res = executeInternal(query, maxEpoch);
+        if (res == null || res.isEmpty())
+            return new Sealed(lowestPeriod, lowestMaxEpoch);
+        // this is the last recorded sealed period *before* the target epoch, so any scan should start at the
+        // *next* period, so we bump both period and epoch by 1
+        long maxPeriod = res.one().getLong("period");
+        return new Sealed(maxPeriod + 1, maxEpoch + 1);
+    }
+
+    public static Sealed getLastSealedPeriod()
+    {
+        String query = String.format("SELECT epoch, period FROM %s.%s WHERE key = 'latest'", SchemaConstants.SYSTEM_KEYSPACE_NAME, LAST_SEALED_PERIOD_TABLE_NAME);
+        UntypedResultSet res = executeInternal(query);
+        if (res == null || res.isEmpty())
+            return Sealed.EMPTY;
+        long epoch = res.one().getLong("epoch");
+        long period = res.one().getLong("period");
+        return new Sealed(period, Epoch.create(epoch));
+    }
+
+    public static void sealPeriod(long period, Epoch epoch)
+    {
+        String query = String.format("INSERT INTO %s.%s (max_epoch, period) VALUES (?,?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, SEALED_PERIODS_TABLE_NAME);
+        executeInternal(query, epoch.getEpoch(), period);
+        query = String.format("UPDATE %s.%s SET period = ?, epoch = ? WHERE key = 'latest'", SchemaConstants.SYSTEM_KEYSPACE_NAME, LAST_SEALED_PERIOD_TABLE_NAME);
+        executeInternal(query, period, epoch.getEpoch());
+    }
+
+    public static Map<InetAddressAndPort, EndpointState> peerEndpointStates()
+    {
+        Map<InetAddressAndPort, EndpointState> epstates = new HashMap<>();
+        VersionedValue.VersionedValueFactory vf = StorageService.instance.valueFactory;
+        String query = String.format("select * from %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, PEERS_V2);
+        UntypedResultSet res = executeInternal(query);
+        for (UntypedResultSet.Row row : res)
+        {
+            EndpointState epstate = new EndpointState(new HeartBeatState(0, 0));
+            InetAddressAndPort endpoint = InetAddressAndPort.getByAddressOverrideDefaults(row.getInetAddress("peer"), row.getInt("peer_port"));
+            epstate.addApplicationState(DC, vf.datacenter(row.getString("data_center")));
+            epstate.addApplicationState(RACK, vf.rack(row.getString("rack")));
+            epstate.addApplicationState(RELEASE_VERSION, vf.releaseVersion(row.getString("release_version")));
+            epstate.addApplicationState(HOST_ID, vf.hostId(row.getUUID("host_id")));
+            Collection<Token> tokens = deserializeTokens(row.getSet("tokens", UTF8Type.instance));
+            epstate.addApplicationState(STATUS_WITH_PORT, vf.normal(tokens));
+            epstate.addApplicationState(TOKENS, vf.tokens(tokens));
+
+            if (row.has("preferred_ip"))
+            {
+                epstate.addApplicationState(INTERNAL_ADDRESS_AND_PORT,
+                                            vf.internalAddressAndPort(InetAddressAndPort.getByAddressOverrideDefaults(row.getInetAddress("preferred_ip"),
+                                                                                                                      row.getInt("preferred_port"))));
+            }
+
+            if (row.has("native_ip"))
+            {
+                epstate.addApplicationState(NATIVE_ADDRESS_AND_PORT,
+                                            vf.nativeaddressAndPort(InetAddressAndPort.getByAddressOverrideDefaults(row.getInetAddress("native_ip"),
+                                                                                                                    row.getInt("native_port"))));
+            }
+
+            epstates.put(endpoint, epstate);
+        }
+        return epstates;
+    }
 }

src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java

@@ -320,11 +320,12 @@ public abstract class AbstractCommitLogSegmentManager
     void forceRecycleAll(Collection<TableId> droppedTables)
     {
         List<CommitLogSegment> segmentsToRecycle = new ArrayList<>(activeSegments);
-        CommitLogSegment last = segmentsToRecycle.get(segmentsToRecycle.size() - 1);
+        CommitLogSegment last = segmentsToRecycle.isEmpty() ? null : segmentsToRecycle.get(segmentsToRecycle.size() - 1);
         advanceAllocatingFrom(last);
 
         // wait for the commit log modifications
-        last.waitForModifications();
+        if (last != null)
+            last.waitForModifications();
 
         // make sure the writes have materialized inside of the memtables by waiting for all outstanding writes
         // to complete
@@ -350,7 +351,7 @@ public abstract class AbstractCommitLogSegmentManager
             }
 
             CommitLogSegment first;
-            if ((first = activeSegments.peek()) != null && first.id <= last.id)
+            if ((first = activeSegments.peek()) != null && last != null && first.id <= last.id)
                 logger.error("Failed to force-recycle all segments; at least one segment is still in use with dirty CFs.");
         }
         catch (Throwable t)

src/java/org/apache/cassandra/db/compaction/CompactionManager.java

@@ -96,6 +96,7 @@ import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.apache.cassandra.metrics.CompactionMetrics;
 import org.apache.cassandra.metrics.TableMetrics;
@@ -113,10 +114,13 @@ import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.WrappedRunnable;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.apache.cassandra.utils.concurrent.Refs;
 
+
 import static java.util.Collections.singleton;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.concurrent.FutureTask.callable;
@@ -617,13 +621,26 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
         assert !cfStore.isIndex();
         Keyspace keyspace = cfStore.keyspace;
 
-        // if local ranges is empty, it means no data should remain
+        if (!StorageService.instance.isJoined())
-        final RangesAtEndpoint replicas = StorageService.instance.getLocalReplicas(keyspace.getName());
+        {
-        final Set<Range<Token>> allRanges = replicas.ranges();
+            logger.info("Cleanup cannot run before a node has joined the ring");
-        final Set<Range<Token>> transientRanges = replicas.onlyTransient().ranges();
+            return AllSSTableOpStatus.ABORTED;
-        final Set<Range<Token>> fullRanges = replicas.onlyFull().ranges();
+        }
+        if (cfStore.keyspace.getMetadata().params.replication.isMeta())
+            return AllSSTableOpStatus.SUCCESSFUL; // todo - we probably want to be able to cleanup MetaStrategy keyspaces
         final boolean hasIndexes = cfStore.indexManager.hasIndexes();
 
+        // if local ranges is empty, it means no data should remain
+        // we only consider write placements during cleanup as range movements always ensure
+        // overlap between new replicas accepting reads and old replicas accepting writes
+        ClusterMetadata cm = ClusterMetadata.current();
+        DataPlacement placement = cm.placements.get(keyspace.getMetadata().params.replication);
+        InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+        RangesAtEndpoint localWrites = placement.writes.byEndpoint().get(local);
+        final Set<Range<Token>> allRanges = new HashSet<>(localWrites.ranges());
+        final Set<Range<Token>> transientRanges = new HashSet<>(localWrites.onlyTransient().ranges());
+        final Set<Range<Token>> fullRanges = new HashSet<>(localWrites.onlyFull().ranges());
+
         return parallelAllSSTableOperation(cfStore, new OneSSTableOperation()
         {
             @Override
@@ -665,7 +682,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
             public void execute(LifecycleTransaction txn) throws IOException
             {
                 CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, allRanges, transientRanges, txn.onlyOne().isRepaired(), FBUtilities.nowInSeconds());
-                doCleanupOne(cfStore, txn, cleanupStrategy, replicas.ranges(), hasIndexes);
+                doCleanupOne(cfStore, txn, cleanupStrategy, allRanges, hasIndexes);
             }
         }, jobs, OperationType.CLEANUP);
     }
@@ -1291,7 +1308,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
     /* Used in tests. */
     public void disableAutoCompaction()
     {
-        for (String ksname : Schema.instance.distributedKeyspaces().names())
+        for (String ksname : Schema.instance.getKeyspaces())
         {
             for (ColumnFamilyStore cfs : Keyspace.open(ksname).getColumnFamilyStores())
                 cfs.disableAutoCompaction();

src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java

@@ -70,6 +70,7 @@ import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.notifications.INotification;
 import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.InitialSSTableAddedNotification;
 import org.apache.cassandra.notifications.SSTableAddedNotification;
 import org.apache.cassandra.notifications.SSTableDeletingNotification;
 import org.apache.cassandra.notifications.SSTableListChangedNotification;
@@ -890,6 +891,11 @@ public class CompactionStrategyManager implements INotificationConsumer
                 SSTableAddedNotification flushedNotification = (SSTableAddedNotification) notification;
                 handleFlushNotification(flushedNotification.added);
             }
+            else if (notification instanceof InitialSSTableAddedNotification)
+            {
+                InitialSSTableAddedNotification flushedNotification = (InitialSSTableAddedNotification) notification;
+                handleFlushNotification(flushedNotification.added);
+            }
             else if (notification instanceof SSTableListChangedNotification)
             {
                 SSTableListChangedNotification listChangedNotification = (SSTableListChangedNotification) notification;

src/java/org/apache/cassandra/db/compaction/ShardManagerNoDisks.java

@@ -55,8 +55,8 @@ public class ShardManagerNoDisks implements ShardManager
 
     public boolean isOutOfDate(long ringVersion)
     {
-        return ringVersion != localRanges.ringVersion &&
+        return ringVersion != localRanges.ringVersion.getEpoch() &&
-               localRanges.ringVersion != ColumnFamilyStore.RING_VERSION_IRRELEVANT;
+               !localRanges.ringVersion.is(ColumnFamilyStore.RING_VERSION_IRRELEVANT);
     }
 
     @Override

src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java

@@ -52,7 +52,7 @@ import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Overlaps;
@@ -295,13 +295,14 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
 
     private void maybeUpdateShardManager()
     {
-        if (shardManager != null && !shardManager.isOutOfDate(StorageService.instance.getTokenMetadata().getRingVersion()))
+        // TODO - modify ShardManager::isOutOfDate to take an Epoch
+        if (shardManager != null && !shardManager.isOutOfDate(ClusterMetadata.current().epoch.getEpoch()))
             return; // the disk boundaries (and thus the local ranges too) have not changed since the last time we calculated
 
         synchronized (this)
         {
             // Recheck after entering critical section, another thread may have beaten us to it.
-            while (shardManager == null || shardManager.isOutOfDate(StorageService.instance.getTokenMetadata().getRingVersion()))
+            while (shardManager == null || shardManager.isOutOfDate(ClusterMetadata.current().epoch.getEpoch()))
                 shardManager = ShardManager.create(cfs);
             // Note: this can just as well be done without the synchronization (races would be benign, just doing some
             // redundant work). For the current usages of this blocking is fine and expected to perform no worse.

src/java/org/apache/cassandra/db/filter/RowFilter.java

@@ -65,6 +65,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
     private static final Logger logger = LoggerFactory.getLogger(RowFilter.class);
 
     public static final Serializer serializer = new Serializer();
+    public static final RowFilter NONE = new CQLFilter(Collections.emptyList());
 
     protected final List<Expression> expressions;
 

src/java/org/apache/cassandra/db/guardrails/Guardrail.java

@@ -95,7 +95,7 @@ public abstract class Guardrail
      */
     public boolean enabled(@Nullable ClientState state)
     {
-        return DatabaseDescriptor.isDaemonInitialized() && (state == null || state.isOrdinaryUser());
+        return DatabaseDescriptor.isDaemonInitialized() && (state == null || (state.isOrdinaryUser() && state.applyGuardrails()));
     }
 
     protected void warn(String message)

src/java/org/apache/cassandra/db/lifecycle/Tracker.java

@@ -230,9 +230,15 @@ public class Tracker
         addSSTablesInternal(sstables, true, false, true);
     }
 
-    public void addInitialSSTablesWithoutUpdatingSize(Iterable<SSTableReader> sstables)
+    public void addInitialSSTablesWithoutUpdatingSize(Iterable<SSTableReader> sstables, ColumnFamilyStore cfs)
     {
-        addSSTablesInternal(sstables, true, false, false);
+        if (!isDummy())
+        {
+            for (SSTableReader reader : sstables)
+                reader.setupOnline();
+        }
+        apply(updateLiveSet(emptySet(), sstables));
+        notifyAdded(sstables, true);
     }
 
     public void updateInitialSSTableSize(Iterable<SSTableReader> sstables)

src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
@@ -128,13 +129,13 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
     }
 
     @Override
-    public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason)
+    public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason, TableMetadata latest)
     {
         switch (reason)
         {
         case SCHEMA_CHANGE:
-            return initialComparator != metadata().comparator // If the CF comparator has changed, because our partitions reference the old one
+            return initialComparator != latest.comparator // If the CF comparator has changed, because our partitions reference the old one
-                   || !initialFactory.equals(metadata().params.memtable.factory()); // If a different type of memtable is requested
+                   || !initialFactory.equals(latest.params.memtable.factory()); // If a different type of memtable is requested
         case OWNED_RANGES_CHANGE:
             return false; // by default we don't use the local ranges, thus this has no effect
         default:

src/java/org/apache/cassandra/db/memtable/Memtable.java

@@ -399,10 +399,19 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
      * - SNAPSHOT will be followed by performSnapshot().
      * - STREAMING/REPAIR will be followed by creating a FlushSet for the streamed/repaired ranges. This data will be
      *   used to create sstables, which will be streamed and then deleted.
+     * The table metadata is supplied explicitly as this might not be the same as the current published metadata for
+     * the table. When applying a schema change, the ColumnFamilyStore instance is reloaded using the new table metadata
+     * before the Schema registry is updated. The memtable needs to examine the new metadata in order to determine
+     * whether the changes warrant a switch.
      * This will not be called to perform truncation or drop (in that case the memtable is unconditionally dropped),
      * but a flush may nevertheless be requested in that case to prepare a snapshot.
      */
-    boolean shouldSwitch(ColumnFamilyStore.FlushReason reason);
+    boolean shouldSwitch(ColumnFamilyStore.FlushReason reason, TableMetadata latest);
+
+    default boolean shouldSwitch(ColumnFamilyStore.FlushReason reason)
+    {
+        return shouldSwitch(reason, metadata());
+    }
 
     /**
      * Called when the table's metadata is updated. The memtable's metadata reference now points to the new version.

src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java

@@ -25,6 +25,7 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.tcm.Epoch;
 
 /**
  * Holds boundaries (tokens) used to map a particular token (so partition key) to a shard id.
@@ -43,21 +44,21 @@ public class ShardBoundaries
     // - there is only 1 shard configured
     // - the default partitioner doesn't support splitting
     // - the keyspace is local system keyspace
-    public static final ShardBoundaries NONE = new ShardBoundaries(EMPTY_TOKEN_ARRAY, -1);
+    public static final ShardBoundaries NONE = new ShardBoundaries(EMPTY_TOKEN_ARRAY, Epoch.EMPTY);
 
     private final Token[] boundaries;
-    public final long ringVersion;
+    public final Epoch epoch;
 
     @VisibleForTesting
-    public ShardBoundaries(Token[] boundaries, long ringVersion)
+    public ShardBoundaries(Token[] boundaries, Epoch epoch)
     {
         this.boundaries = boundaries;
-        this.ringVersion = ringVersion;
+        this.epoch = epoch;
     }
 
-    public ShardBoundaries(List<Token> boundaries, long ringVersion)
+    public ShardBoundaries(List<Token> boundaries, Epoch epoch)
     {
-        this(boundaries.toArray(EMPTY_TOKEN_ARRAY), ringVersion);
+        this(boundaries.toArray(EMPTY_TOKEN_ARRAY), epoch);
     }
 
     /**

src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java

@@ -35,11 +35,17 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.exceptions.CoordinatorBehindException;
+import org.apache.cassandra.exceptions.UnknownTableException;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.metrics.TCMMetrics;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
@@ -73,10 +79,12 @@ public class PartitionUpdate extends AbstractBTreePartition
     private final BTreePartitionData holder;
     private final DeletionInfo deletionInfo;
     private final TableMetadata metadata;
+    public final Epoch serializedAtEpoch;
 
     private final boolean canHaveShadowedData;
 
     private PartitionUpdate(TableMetadata metadata,
+                            Epoch serializedAtEpoch,
                             DecoratedKey key,
                             BTreePartitionData holder,
                             MutableDeletionInfo deletionInfo,
@@ -87,6 +95,7 @@ public class PartitionUpdate extends AbstractBTreePartition
         this.holder = holder;
         this.deletionInfo = deletionInfo;
         this.canHaveShadowedData = canHaveShadowedData;
+        this.serializedAtEpoch = serializedAtEpoch;
     }
 
     /**
@@ -101,7 +110,7 @@ public class PartitionUpdate extends AbstractBTreePartition
     {
         MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
         BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
+        return new PartitionUpdate(metadata, metadata.epoch, key, holder, deletionInfo, false);
     }
 
     /**
@@ -118,7 +127,7 @@ public class PartitionUpdate extends AbstractBTreePartition
     {
         MutableDeletionInfo deletionInfo = new MutableDeletionInfo(timestamp, nowInSec);
         BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
+        return new PartitionUpdate(metadata, metadata.epoch, key, holder, deletionInfo, false);
     }
 
     /**
@@ -144,7 +153,7 @@ public class PartitionUpdate extends AbstractBTreePartition
             staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow,
             EncodingStats.NO_STATS
         );
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
+        return new PartitionUpdate(metadata, metadata.epoch, key, holder, deletionInfo, false);
     }
 
     /**
@@ -191,7 +200,7 @@ public class PartitionUpdate extends AbstractBTreePartition
         iterator = UnfilteredRowIterators.withOnlyQueriedData(iterator, filter);
         BTreePartitionData holder = build(iterator, 16);
         MutableDeletionInfo deletionInfo = (MutableDeletionInfo) holder.deletionInfo;
-        return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
+        return new PartitionUpdate(iterator.metadata(), iterator.metadata().epoch, iterator.partitionKey(), holder, deletionInfo, false);
     }
 
     /**
@@ -210,7 +219,7 @@ public class PartitionUpdate extends AbstractBTreePartition
         iterator = RowIterators.withOnlyQueriedData(iterator, filter);
         MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
         BTreePartitionData holder = build(iterator, deletionInfo, true);
-        return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
+        return new PartitionUpdate(iterator.metadata(), iterator.metadata().epoch, iterator.partitionKey(), holder, deletionInfo, false);
     }
 
 
@@ -223,7 +232,7 @@ public class PartitionUpdate extends AbstractBTreePartition
                 columnSet.add(column.column());
 
         RegularAndStaticColumns columns = RegularAndStaticColumns.builder().addAll(columnSet).build();
-        return new PartitionUpdate(this.metadata, this.partitionKey, this.holder.withColumns(columns), this.deletionInfo.mutableCopy(), false);
+        return new PartitionUpdate(this.metadata, this.metadata.epoch, this.partitionKey, this.holder.withColumns(columns), this.deletionInfo.mutableCopy(), false);
     }
 
 
@@ -539,7 +548,14 @@ public class PartitionUpdate extends AbstractBTreePartition
      */
     public static SimpleBuilder simpleBuilder(TableMetadata metadata, Object... partitionKeyValues)
     {
-        return new SimpleBuilders.PartitionUpdateBuilder(metadata, partitionKeyValues);
+        // Here we dereference the current version of the supplied TableMetadata. The reason for this is that in some
+        // places we still reference static TableMetadata instances.
+        // For instance, TraceKeyspace contains Sessions & Events static members which are created at startup when the
+        // current epoch is Epoch.EMPTY. These are used to construct mutations when tracing is enabled and when the
+        // mutations are serialised and sent between replica & coordinator the epoch comparisons in PartitionUpdate
+        // deserializer trigger an IncompatibleSchemaException.
+        // TODO ultimately remove the use of static TableMetadata instances in System/Tracing/Auth keyspaces.
+        return new SimpleBuilders.PartitionUpdateBuilder(metadata.ref.get(), partitionKeyValues);
     }
 
     public void validateIndexedColumns()
@@ -554,7 +570,7 @@ public class PartitionUpdate extends AbstractBTreePartition
                                                   MutableDeletionInfo deletionInfo,
                                                   boolean canHaveShadowedData)
     {
-        return new PartitionUpdate(metadata, key, holder, deletionInfo, canHaveShadowedData);
+        return new PartitionUpdate(metadata, metadata.epoch, key, holder, deletionInfo, canHaveShadowedData);
     }
 
     /**
@@ -713,24 +729,45 @@ public class PartitionUpdate extends AbstractBTreePartition
                 assert !iter.isReverseOrder();
 
                 update.metadata.id.serialize(out);
+                if (version >= MessagingService.VERSION_50)
+                    Epoch.serializer.serialize(update.metadata.epoch != null ? update.metadata.epoch : Epoch.EMPTY, out);
                 UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, update.rowCount());
             }
         }
 
         public PartitionUpdate deserialize(DataInputPlus in, int version, DeserializationHelper.Flag flag) throws IOException
         {
-            TableMetadata metadata = Schema.instance.getExistingTableMetadata(TableId.deserialize(in));
+            TableId tableId = TableId.deserialize(in);
-            UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, flag);
+            Epoch remoteVersion = null;
+            if (version >= MessagingService.VERSION_50)
+                remoteVersion = Epoch.serializer.deserialize(in);
+            TableMetadata tableMetadata;
+            try
+            {
+                tableMetadata = Schema.instance.getExistingTableMetadata(tableId);
+            }
+            catch (UnknownTableException e)
+            {
+                ClusterMetadata metadata = ClusterMetadata.current();
+                Epoch localCurrentEpoch = metadata.epoch;
+                if (remoteVersion != null && localCurrentEpoch.isAfter(remoteVersion))
+                {
+                    TCMMetrics.instance.coordinatorBehindSchema.mark();
+                    throw new CoordinatorBehindException(e.getMessage(), e);
+                }
+                throw e;
+            }
+            UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(tableMetadata, null, in, version, flag);
             if (header.isEmpty)
-                return emptyUpdate(metadata, header.key);
+                return emptyUpdate(tableMetadata, header.key);
 
             assert !header.isReversed;
             assert header.rowEstimate >= 0;
 
-            MutableDeletionInfo.Builder deletionBuilder = MutableDeletionInfo.builder(header.partitionDeletion, metadata.comparator, false);
+            MutableDeletionInfo.Builder deletionBuilder = MutableDeletionInfo.builder(header.partitionDeletion, tableMetadata.comparator, false);
             Object[] rows;
             try (BTree.FastBuilder<Row> builder = BTree.fastBuilder();
-                 UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, flag, header))
+                 UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, tableMetadata, flag, header))
             {
                 while (partition.hasNext())
                 {
@@ -744,19 +781,27 @@ public class PartitionUpdate extends AbstractBTreePartition
             }
 
             MutableDeletionInfo deletionInfo = deletionBuilder.build();
-            return new PartitionUpdate(metadata,
+            return new PartitionUpdate(tableMetadata,
+                                       remoteVersion,
                                        header.key,
                                        new BTreePartitionData(header.sHeader.columns(), rows, deletionInfo, header.staticRow, header.sHeader.stats()),
                                        deletionInfo,
                                        false);
         }
 
-        public static boolean isEmpty(ByteBuffer in, DeserializationHelper.Flag flag, DecoratedKey key) throws IOException
+        public static boolean isEmpty(ByteBuffer in, DeserializationHelper.Flag flag, DecoratedKey key, int version) throws IOException
         {
             int position = in.position();
             position += 16; // CFMetaData.serializer.deserialize(in, version);
             if (position >= in.limit())
                 throw new EOFException();
+
+            if (version >= MessagingService.VERSION_50)
+            {
+                long epoch = VIntCoding.getUnsignedVInt(in, position);
+                position += VIntCoding.computeVIntSize(epoch);
+            }
+
             // DecoratedKey key = metadata.decorateKey(ByteBufferUtil.readWithVIntLength(in));
             int keyLength = VIntCoding.getUnsignedVInt32(in, position);
             position += keyLength + VIntCoding.computeUnsignedVIntSize(keyLength);
@@ -771,6 +816,7 @@ public class PartitionUpdate extends AbstractBTreePartition
             try (UnfilteredRowIterator iter = update.unfilteredIterator())
             {
                 return update.metadata.id.serializedSize()
+                     + (version >= MessagingService.VERSION_50 ? Epoch.serializer.serializedSize(update.metadata.epoch) : 0)
                      + UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, null, version, update.rowCount());
             }
         }
@@ -964,6 +1010,7 @@ public class PartitionUpdate extends AbstractBTreePartition
 
             isBuilt = true;
             return new PartitionUpdate(metadata,
+                                       metadata.epoch,
                                        partitionKey(),
                                        new BTreePartitionData(columns,
                                                               merged,

src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java

@@ -74,8 +74,8 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader
         try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance))
         {
             TrackedDataInputPlus in = new TrackedDataInputPlus(cis);
-            deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
             writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format);
+            deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()), session, writer);
             String filename = writer.getFilename();
             String sectionName = filename + '-' + fileSeqNum;
             int sectionIdx = 0;

src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java

@@ -20,6 +20,9 @@ package org.apache.cassandra.db.streaming;
 import java.io.IOError;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.UnmodifiableIterator;
@@ -38,6 +41,8 @@ import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.UnknownColumnException;
 import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
@@ -47,15 +52,19 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.TrackedDataInputPlus;
+import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.streaming.ProgressInfo;
+import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException;
 import org.apache.cassandra.streaming.StreamReceiver;
 import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.compress.StreamCompressionInputStream;
 import org.apache.cassandra.streaming.messages.StreamMessageHeader;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.net.MessagingService.current_version;
@@ -66,6 +75,7 @@ import static org.apache.cassandra.net.MessagingService.current_version;
 public class CassandraStreamReader implements IStreamReader
 {
     private static final Logger logger = LoggerFactory.getLogger(CassandraStreamReader.class);
+    private static final String logMessageTemplate = "[Stream #{}] Received streamed SSTable {} from {} containing key outside valid ranges {}";
     protected final TableId tableId;
     protected final long estimatedKeys;
     protected final Collection<SSTableReader.PartitionPositionBounds> sections;
@@ -121,8 +131,8 @@ public class CassandraStreamReader implements IStreamReader
         try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version))
         {
             TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream);
-            deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
             writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format);
+            deserializer = getDeserializer(cfs.metadata(), in, inputVersion, session, writer);
             String sequenceName = writer.getFilename() + '-' + fileSeqNum;
             long lastBytesRead = 0;
             while (in.getBytesRead() < totalSize)
@@ -149,6 +159,15 @@ public class CassandraStreamReader implements IStreamReader
         }
     }
 
+    protected StreamDeserializer getDeserializer(TableMetadata metadata,
+                                                 TrackedDataInputPlus in,
+                                                 Version inputVersion,
+                                                 StreamSession session,
+                                                 SSTableMultiWriter writer) throws IOException
+    {
+        return new StreamDeserializer(metadata, in, inputVersion, getHeader(metadata), session, writer);
+    }
+
     protected SerializationHeader getHeader(TableMetadata metadata) throws UnknownColumnException
     {
         return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader
@@ -188,29 +207,52 @@ public class CassandraStreamReader implements IStreamReader
         private final SerializationHeader header;
         private final DeserializationHelper helper;
 
-        private DecoratedKey key;
+        private final List<Range<Token>> ownedRanges;
-        private DeletionTime partitionLevelDeletion;
+        private final StreamSession session;
-        private SSTableSimpleIterator iterator;
+        private final SSTableMultiWriter writer;
-        private Row staticRow;
+
+        private int lastCheckedRangeIndex;
+        protected DecoratedKey key;
+        protected DeletionTime partitionLevelDeletion;
+        protected SSTableSimpleIterator iterator;
+        protected Row staticRow;
         private IOException exception;
         private Version version;
 
-        public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header) throws IOException
+        public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header, StreamSession session, SSTableMultiWriter writer) throws IOException
         {
             this.metadata = metadata;
             this.in = in;
             this.helper = new DeserializationHelper(metadata, version.correspondingMessagingVersion(), DeserializationHelper.Flag.PRESERVE_SIZE);
             this.header = header;
             this.version = version;
+            ownedRanges = Range.normalize(StorageService.instance.getLocalAndPendingRanges(metadata.keyspace));
+            lastCheckedRangeIndex = 0;
+
+            this.session = session;
+            this.writer = writer;
         }
 
-        public StreamDeserializer newPartition() throws IOException
+        public UnfilteredRowIterator newPartition() throws IOException
+        {
+            readKey();
+            readPartition();
+            return this;
+        }
+
+        protected void readKey() throws IOException
         {
             key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in));
+            lastCheckedRangeIndex = verifyKeyInOwnedRanges(key,
+                                                           ownedRanges,
+                                                           lastCheckedRangeIndex);
+        }
+
+        protected void readPartition() throws IOException
+        {
             partitionLevelDeletion = DeletionTime.getSerializer(version).deserialize(in);
             iterator = SSTableSimpleIterator.create(metadata, in, header, helper, partitionLevelDeletion);
             staticRow = iterator.readStaticRow();
-            return this;
         }
 
         public TableMetadata metadata()
@@ -291,5 +333,27 @@ public class CassandraStreamReader implements IStreamReader
         public void close()
         {
         }
+
+        private int verifyKeyInOwnedRanges(final DecoratedKey key,
+                                           List<Range<Token>> ownedRanges,
+                                           int lastCheckedRangeIndex)
+        {
+            if (lastCheckedRangeIndex < ownedRanges.size())
+            {
+                ListIterator<Range<Token>> rangesToCheck = ownedRanges.listIterator(lastCheckedRangeIndex);
+                while (rangesToCheck.hasNext())
+                {
+                    Range<Token> range = rangesToCheck.next();
+                    if (range.contains(key.getToken()))
+                        return lastCheckedRangeIndex;
+
+                    lastCheckedRangeIndex++;
+                }
+            }
+
+            StorageMetrics.totalOpsForInvalidToken.inc();
+            NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, session.planId(), writer.getFilename(), session.peer, ownedRanges);
+            throw new StreamReceivedOutOfTokenRangeException(ownedRanges, key, writer.getFilename());
+        }
     }
 }

src/java/org/apache/cassandra/db/view/TableViews.java

@@ -17,7 +17,14 @@
  */
 package org.apache.cassandra.db.view;
 
-import java.util.*;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
@@ -27,14 +34,36 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.ReadQuery;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.StorageProxy;
@@ -57,9 +86,9 @@ public class TableViews extends AbstractCollection<View>
     // list is the best option.
     private final List<View> views = new CopyOnWriteArrayList();
 
-    public TableViews(TableId id)
+    public TableViews(TableMetadata tableMetadata)
     {
-        baseTableMetadata = Schema.instance.getTableMetadataRef(id);
+        baseTableMetadata = tableMetadata.ref;
     }
 
     public boolean hasViews()

src/java/org/apache/cassandra/db/view/ViewBuilder.java

@@ -43,6 +43,7 @@ import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.apache.cassandra.locator.Replicas;
 import org.apache.cassandra.schema.SystemDistributedKeyspace;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -66,7 +67,7 @@ class ViewBuilder
     private final ColumnFamilyStore baseCfs;
     private final View view;
     private final String ksName;
-    private final UUID localHostId = SystemKeyspace.getOrInitializeLocalHostId();
+    private final UUID localHostId;
     private final Set<Range<Token>> builtRanges = Sets.newConcurrentHashSet();
     private final Map<Range<Token>, Pair<Token, Long>> pendingRanges = Maps.newConcurrentMap();
     private final Set<ViewBuilderTask> tasks = Sets.newConcurrentHashSet();
@@ -79,6 +80,7 @@ class ViewBuilder
         this.baseCfs = baseCfs;
         this.view = view;
         ksName = baseCfs.metadata.keyspace;
+        this.localHostId = ClusterMetadata.current().myNodeId().toUUID();
     }
 
     public void start()

src/java/org/apache/cassandra/db/view/ViewBuilderTask.java

@@ -110,7 +110,7 @@ public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<L
              UnfilteredRowIterator data = UnfilteredPartitionIterators.getOnlyElement(command.executeLocally(orderGroup), command))
         {
             Iterator<Collection<Mutation>> mutations = baseCfs.keyspace.viewManager
-                                                       .forTable(baseCfs.metadata.id)
+                                                       .forTable(baseCfs.metadata.get())
                                                        .generateViewUpdates(Collections.singleton(view), data, empty, nowInSec, true);
 
             AtomicLong noBase = new AtomicLong(Long.MAX_VALUE);

src/java/org/apache/cassandra/db/view/ViewManager.java

@@ -28,12 +28,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.*;
-import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.schema.SystemDistributedKeyspace;
-import org.apache.cassandra.schema.Views;
 import org.apache.cassandra.service.StorageService;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ENABLE_COORDINATOR_BATCHLOG;
@@ -84,7 +81,7 @@ public class ViewManager
                 if (coordinatorBatchlog && keyspace.getReplicationStrategy().getReplicationFactor().allReplicas == 1)
                     continue;
 
-                if (!forTable(update.metadata().id).updatedViews(update).isEmpty())
+                if (!forTable(update.metadata()).updatedViews(update).isEmpty())
                     return true;
             }
         }
@@ -97,9 +94,9 @@ public class ViewManager
         return viewsByName.values();
     }
 
-    public void reload(boolean buildAllViews)
+    public void reload(KeyspaceMetadata keyspaceMetadata)
     {
-        Views views = keyspace.getMetadata().views;
+        Views views = keyspaceMetadata.views;
         Map<String, ViewMetadata> newViewsByName = Maps.newHashMapWithExpectedSize(views.size());
         for (ViewMetadata definition : views)
         {
@@ -111,10 +108,16 @@ public class ViewManager
             if (!viewsByName.containsKey(entry.getKey()))
                 addView(entry.getValue());
         }
+    }
 
-        if (!buildAllViews)
+    public void buildViews()
-            return;
+    {
-
+        Views views = keyspace.getMetadata().views;
+        Map<String, ViewMetadata> newViewsByName = Maps.newHashMapWithExpectedSize(views.size());
+        for (ViewMetadata definition : views)
+        {
+            newViewsByName.put(definition.name(), definition);
+        }
         // Building views involves updating view build status in the system_distributed
         // keyspace and therefore it requires ring information. This check prevents builds
         // being submitted when Keyspaces are initialized during CassandraDaemon::setup as
@@ -149,7 +152,7 @@ public class ViewManager
         }
 
         View view = new View(definition, keyspace.getColumnFamilyStore(definition.baseTableId));
-        forTable(view.getDefinition().baseTableId).add(view);
+        forTable(keyspace.getMetadata().tables.getNullable(view.getDefinition().baseTableId)).add(view);
         viewsByName.put(definition.name(), view);
     }
 
@@ -166,7 +169,7 @@ public class ViewManager
             return;
 
         view.stopBuild();
-        forTable(view.getDefinition().baseTableId).removeByName(name);
+        forTable(view.getDefinition().baseTableMetadata()).removeByName(name);
         SystemKeyspace.setViewRemoved(keyspace.getName(), view.name);
         SystemDistributedKeyspace.setViewRemoved(keyspace.getName(), view.name);
     }
@@ -182,13 +185,13 @@ public class ViewManager
             view.build();
     }
 
-    public TableViews forTable(TableId id)
+    public TableViews forTable(TableMetadata metadata)
     {
-        TableViews views = viewsByBaseTable.get(id);
+        TableViews views = viewsByBaseTable.get(metadata.id);
         if (views == null)
         {
-            views = new TableViews(id);
+            views = new TableViews(metadata);
-            TableViews previous = viewsByBaseTable.putIfAbsent(id, views);
+            TableViews previous = viewsByBaseTable.putIfAbsent(metadata.id, views);
             if (previous != null)
                 views = previous;
         }

src/java/org/apache/cassandra/db/view/ViewUtils.java

@@ -24,10 +24,11 @@ import java.util.function.Predicate;
 import com.google.common.collect.Iterables;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.EndpointsForToken;
 import org.apache.cassandra.locator.NetworkTopologyStrategy;
 import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.tcm.ClusterMetadata;
 
 public final class ViewUtils
 {
@@ -57,11 +58,13 @@ public final class ViewUtils
      *
      * @return Optional.empty() if this method is called using a base token which does not belong to this replica
      */
-    public static Optional<Replica> getViewNaturalEndpoint(AbstractReplicationStrategy replicationStrategy, Token baseToken, Token viewToken)
+    public static Optional<Replica> getViewNaturalEndpoint(ClusterMetadata metadata, String keyspace, Token baseToken, Token viewToken)
     {
         String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
-        EndpointsForToken naturalBaseReplicas = replicationStrategy.getNaturalReplicasForToken(baseToken);
+        KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
-        EndpointsForToken naturalViewReplicas = replicationStrategy.getNaturalReplicasForToken(viewToken);
+
+        EndpointsForToken naturalBaseReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(baseToken).get();
+        EndpointsForToken naturalViewReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(viewToken).get();
 
         Optional<Replica> localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isSelf).toJavaUtil();
         if (localReplica.isPresent())
@@ -69,7 +72,7 @@ public final class ViewUtils
 
         // We only select replicas from our own DC
         // TODO: this is poor encapsulation, leaking implementation details of replication strategy
-        Predicate<Replica> isLocalDC = r -> !(replicationStrategy instanceof NetworkTopologyStrategy)
+        Predicate<Replica> isLocalDC = r -> !(keyspaceMetadata.replicationStrategy instanceof NetworkTopologyStrategy)
                 || DatabaseDescriptor.getEndpointSnitch().getDatacenter(r).equals(localDataCenter);
 
         // We have to remove any endpoint which is shared between the base and the view, as it will select itself
@@ -84,7 +87,9 @@ public final class ViewUtils
         // The replication strategy will be the same for the base and the view, as they must belong to the same keyspace.
         // Since the same replication strategy is used, the same placement should be used and we should get the same
         // number of replicas for all of the tokens in the ring.
-        assert baseReplicas.size() == viewReplicas.size() : "Replication strategy should have the same number of endpoints for the base and the view";
+        assert baseReplicas.size() == viewReplicas.size() :
+        String.format("Replication strategy should have the same number of endpoints for the base (%d) and the view (%d)",
+                      baseReplicas.size(), viewReplicas.size());
 
         int baseIdx = -1;
         for (int i=0; i<baseReplicas.size(); i++)

src/java/org/apache/cassandra/db/virtual/ClusterMetadataLogTable.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.virtual;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tcm.Transformation;
+
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.QueryProcessor.execute;
+import static org.apache.cassandra.schema.DistributedMetadataLogKeyspace.TABLE_NAME;
+import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME;
+
+final class ClusterMetadataLogTable extends AbstractVirtualTable
+{
+    private static final String PERIOD = "period";
+    private static final String EPOCH = "epoch";
+    private static final String KIND = "kind";
+    private static final String TRANSFORMATION = "transformation";
+    private static final String ENTRY_ID = "entry_id";
+    private static final String ENTRY_TIME = "entry_time";
+
+    ClusterMetadataLogTable(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, "cluster_metadata_log")
+                           .comment("cluster metadata log")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .partitioner(new LocalPartitioner(LongType.instance))
+                           .addPartitionKeyColumn(PERIOD, LongType.instance)
+                           .addClusteringColumn(EPOCH, LongType.instance)
+                           .addRegularColumn(KIND, UTF8Type.instance)
+                           .addRegularColumn(TRANSFORMATION, UTF8Type.instance)
+                           .addRegularColumn(ENTRY_ID, LongType.instance)
+                           .addRegularColumn(ENTRY_TIME, TimestampType.instance)
+                           .build());
+    }
+
+    @Override
+    public DataSet data()
+    {
+        try
+        {
+            SimpleDataSet result = new SimpleDataSet(metadata());
+            UntypedResultSet res = execute(format("SELECT period, epoch, kind, transformation, entry_id, writetime(kind) as wt " +
+                                                  "FROM %s.%s", METADATA_KEYSPACE_NAME, TABLE_NAME), ConsistencyLevel.QUORUM);
+            for (UntypedResultSet.Row r : res)
+            {
+                Transformation.Kind kind = Transformation.Kind.valueOf(r.getString("kind"));
+                Transformation transformation = kind.fromVersionedBytes(r.getBlob("transformation"));
+
+                result.row(r.getLong("period"), r.getLong("epoch"))
+                      .column(KIND, kind.toString())
+                      .column(TRANSFORMATION, transformation.toString())
+                      .column(ENTRY_ID, r.getLong("entry_id"))
+                      .column(ENTRY_TIME, new Date(r.getLong("wt") / 1000));
+            }
+            return result;
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+}

src/java/org/apache/cassandra/db/virtual/LocalTable.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.util.HashSet;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.Location;
+import org.apache.cassandra.tcm.membership.NodeAddresses;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
+
+public class LocalTable extends AbstractVirtualTable
+{
+    public static final String BOOTSTRAPPED = "bootstrapped";
+    public static final String BROADCAST_ADDRESS = "broadcast_address";
+    public static final String BROADCAST_PORT = "broadcast_port";
+    public static final String CLUSTER_NAME = "cluster_name";
+    public static final String LISTEN_ADDRESS = "listen_address";
+    public static final String LISTEN_PORT = "listen_port";
+    public static final String CQL_VERSION = "cql_version";
+    public static final String DATACENTER = "datacenter";
+    public static final String RACK = "rack";
+    public static final String HOST_ID = "host_id";
+    public static final String GOSSIP_GENERATION = "gossip_generation";
+    public static final String RELEASE_VERSION = "release_version";
+    public static final String NATIVE_ADDRESS = "native_address";
+    public static final String NATIVE_PORT = "native_port";
+    public static final String NATIVE_PROTOCOL_VERSION = "native_protocol_version";
+    public static final String PARTITIONER = "partitioner";
+    public static final String SCHEMA_VERSION = "schema_version";
+    public static final String TOKENS = "tokens";
+    public static final String STATE = "state";
+    public static final String STATUS = "status";
+    public static final String KEY = "local";
+    public static final String TRUNCATED_AT = "truncated_at";
+
+    public LocalTable(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, "local")
+                           .comment("Information about local node")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .partitioner(new LocalPartitioner(InetAddressType.instance))
+                           .addPartitionKeyColumn(KEY, UTF8Type.instance)
+                           .addRegularColumn(BOOTSTRAPPED, UTF8Type.instance)
+                           .addRegularColumn(BROADCAST_ADDRESS, InetAddressType.instance)
+                           .addRegularColumn(BROADCAST_PORT, Int32Type.instance)
+                           .addRegularColumn(CLUSTER_NAME, UTF8Type.instance)
+                           .addRegularColumn(CQL_VERSION, UTF8Type.instance)
+                           .addRegularColumn(DATACENTER, UTF8Type.instance)
+                           .addRegularColumn(GOSSIP_GENERATION, Int32Type.instance)
+                           .addRegularColumn(HOST_ID, UUIDType.instance)
+                           .addRegularColumn(LISTEN_ADDRESS, InetAddressType.instance)
+                           .addRegularColumn(LISTEN_PORT, Int32Type.instance)
+                           .addRegularColumn(NATIVE_ADDRESS, InetAddressType.instance)
+                           .addRegularColumn(NATIVE_PORT, Int32Type.instance)
+                           .addRegularColumn(NATIVE_PROTOCOL_VERSION, UTF8Type.instance)
+                           .addRegularColumn(PARTITIONER, UTF8Type.instance)
+                           .addRegularColumn(RACK, UTF8Type.instance)
+                           .addRegularColumn(RELEASE_VERSION, UTF8Type.instance)
+                           .addRegularColumn(SCHEMA_VERSION, UUIDType.instance)
+                           .addRegularColumn(STATE, UTF8Type.instance)
+                           .addRegularColumn(STATUS, UTF8Type.instance)
+                           .addRegularColumn(TOKENS, SetType.getInstance(UTF8Type.instance, false))
+//                           .addRegularColumn(TRUNCATED_AT, MapType.getInstance(UUIDType.instance, UTF8Type.instance, false)) todo?
+                           .build());
+    }
+
+    public DataSet data()
+    {
+        SimpleDataSet result = new SimpleDataSet(metadata());
+
+        ClusterMetadata cm = ClusterMetadata.current();
+        NodeId peer = cm.myNodeId();
+        NodeState nodeState = cm.directory.peerState(peer);
+        NodeAddresses addresses = cm.directory.getNodeAddresses(peer);
+        Location location = cm.directory.location(peer);
+        result.row(KEY)
+              .column(BOOTSTRAPPED, SystemKeyspace.BootstrapState.fromNodeState(nodeState).toString())
+              .column(BROADCAST_ADDRESS, addresses.broadcastAddress.getAddress())
+              .column(BROADCAST_PORT, addresses.broadcastAddress.getPort())
+              .column(CLUSTER_NAME, DatabaseDescriptor.getClusterName())
+              .column(CQL_VERSION, QueryProcessor.CQL_VERSION.toString())
+              .column(DATACENTER, location.datacenter)
+              .column(GOSSIP_GENERATION, Gossiper.instance.getCurrentGenerationNumber(FBUtilities.getBroadcastAddressAndPort()))
+              .column(HOST_ID, peer.toUUID())
+              .column(LISTEN_ADDRESS, addresses.localAddress.getAddress())
+              .column(LISTEN_PORT, addresses.localAddress.getPort())
+              .column(NATIVE_ADDRESS, addresses.nativeAddress.getAddress())
+              .column(NATIVE_PORT, addresses.nativeAddress.getPort())
+              .column(NATIVE_PROTOCOL_VERSION, String.valueOf(ProtocolVersion.CURRENT.asInt()))
+              .column(PARTITIONER, cm.partitioner.getClass().getName())
+              .column(RACK, location.rack)
+              .column(RELEASE_VERSION, cm.directory.version(peer).cassandraVersion.toString())
+              .column(SCHEMA_VERSION, cm.schema.getVersion())
+              .column(STATE, cm.directory.peerState(peer).toString())
+              .column(STATUS, status(cm))
+              .column(TOKENS, new HashSet<>(cm.tokenMap.tokens(peer).stream().map((token) -> token.getToken().getTokenValue().toString()).collect(Collectors.toList())));
+              //.column(TRUNCATED_AT, status(cm)); // todo?
+
+        return result;
+    }
+
+    private static String status(ClusterMetadata cm)
+    {
+        if (StorageService.instance.isDraining())
+            return StorageService.Mode.DRAINING.toString();
+        if (StorageService.instance.isDrained())
+            return StorageService.Mode.DRAINED.toString();
+        return cm.directory.peerState(getBroadcastAddressAndPort()).toString();
+    }
+}

src/java/org/apache/cassandra/db/virtual/PeersTable.java

@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.virtual;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.SetType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.Location;
+import org.apache.cassandra.tcm.membership.NodeAddresses;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.db.SystemKeyspace.LEGACY_PEERS;
+import static org.apache.cassandra.db.SystemKeyspace.PEERS_V2;
+import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME;
+
+public class PeersTable extends AbstractVirtualTable
+{
+
+    public static String PEER = "peer";
+    public static String PEER_PORT = "peer_port";
+    public static String DATA_CENTER = "data_center";
+    public static String HOST_ID = "host_id";
+    public static String PREFERRED_IP = "preferred_ip";
+    public static String PREFERRED_PORT = "preferred_port";
+    public static String RACK = "rack";
+    public static String RELEASE_VERSION = "release_version";
+    public static String NATIVE_ADDRESS = "native_address";
+    public static String NATIVE_PORT = "native_port";
+    public static String SCHEMA_VERSION = "schema_version";
+    public static String TOKENS = "tokens";
+    public static String STATE = "state";
+
+    public PeersTable(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, "peers")
+                           .comment("Peers")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .partitioner(new LocalPartitioner(InetAddressType.instance))
+                           .addPartitionKeyColumn(PEER, InetAddressType.instance)
+                           .addClusteringColumn(PEER_PORT, Int32Type.instance)
+                           .addRegularColumn(DATA_CENTER, UTF8Type.instance)
+                           .addRegularColumn(RACK, UTF8Type.instance)
+                           .addRegularColumn(HOST_ID, UUIDType.instance)
+                           .addRegularColumn(PREFERRED_IP, InetAddressType.instance)
+                           .addRegularColumn(PREFERRED_PORT, Int32Type.instance)
+                           .addRegularColumn(NATIVE_ADDRESS, InetAddressType.instance)
+                           .addRegularColumn(NATIVE_PORT, Int32Type.instance)
+                           .addRegularColumn(RELEASE_VERSION, UTF8Type.instance)
+                           .addRegularColumn(SCHEMA_VERSION, UUIDType.instance)
+                           .addRegularColumn(STATE, UTF8Type.instance)
+                           .addRegularColumn(TOKENS, SetType.getInstance(UTF8Type.instance, false))
+                           .build());
+    }
+
+    public DataSet data()
+    {
+        SimpleDataSet result = new SimpleDataSet(metadata());
+
+        ClusterMetadata metadata = ClusterMetadata.current();
+        for (InetAddressAndPort addr : metadata.directory.allJoinedEndpoints())
+        {
+            NodeId peer = metadata.directory.peerId(addr);
+
+            NodeAddresses addresses = metadata.directory.getNodeAddresses(peer);
+            result.row(addr.getAddress(), addr.getPort())
+                  .column(DATA_CENTER, metadata.directory.location(peer).datacenter)
+                  .column(RACK, metadata.directory.location(peer).rack)
+                  .column(HOST_ID, peer.toUUID())
+                  .column(PREFERRED_IP, addresses.broadcastAddress.getAddress())
+                  .column(PREFERRED_PORT, addresses.broadcastAddress.getPort())
+                  .column(NATIVE_ADDRESS, addresses.nativeAddress.getAddress())
+                  .column(NATIVE_PORT, addresses.nativeAddress.getPort())
+                  .column(RELEASE_VERSION, metadata.directory.version(peer).cassandraVersion.toString())
+                  .column(SCHEMA_VERSION, Schema.instance.getVersion()) //TODO
+                  .column(STATE, metadata.directory.peerState(peer).toString())
+                  .column(TOKENS, new HashSet<>(metadata.tokenMap.tokens(peer).stream().map((token) -> token.getToken().getTokenValue().toString()).collect(Collectors.toList())));
+        }
+
+        return result;
+    }
+
+    public static void initializeLegacyPeerTables(ClusterMetadata prev, ClusterMetadata next)
+    {
+        QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SYSTEM_KEYSPACE_NAME, PEERS_V2));
+        QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SYSTEM_KEYSPACE_NAME, LEGACY_PEERS));
+
+        for (NodeId nodeId : next.directory.peerIds())
+            updateLegacyPeerTable(nodeId, prev, next);
+    }
+
+    private static String peers_v2_query = "INSERT INTO %s.%s ("
+                                            + "peer, peer_port, "
+                                            + "preferred_ip, preferred_port, "
+                                            + "native_address, native_port, "
+                                            + "data_center, rack, "
+                                            + "host_id, "
+                                            + "release_version, "
+                                            + "schema_version,"
+                                            + "tokens) " +
+                                            "VALUES " +
+                                            "(?,?,?,?,?,?,?,?,?,?,?,?)";
+
+    private static String legacy_peers_query = "INSERT INTO %s.%s ("
+                                               + "peer, preferred_ip, rpc_address, "
+                                               + "data_center, rack, "
+                                               + "host_id, "
+                                               + "release_version, "
+                                               + "schema_version,"
+                                               + "tokens) " +
+                                               "VALUES " +
+                                               "(?,?,?,?,?,?,?,?,?)";
+
+    private static String peers_delete_query = "DELETE FROM %s.%s WHERE peer=? and peer_port=?";
+    private static String legacy_peers_delete_query = "DELETE FROM %s.%s WHERE peer=?";
+
+    private static final Logger logger = LoggerFactory.getLogger(PeersTable.class);
+    public static void updateLegacyPeerTable(NodeId nodeId, ClusterMetadata prev, ClusterMetadata next)
+    {
+        if (nodeId.equals(next.directory.peerId(FBUtilities.getBroadcastAddressAndPort())))
+            return;
+
+        if (next.directory.peerState(nodeId) == null || next.directory.peerState(nodeId) == NodeState.LEFT)
+        {
+            NodeAddresses addresses = prev.directory.getNodeAddresses(nodeId);
+            logger.debug("Purging {} from system.peers_v2 table", addresses);
+            QueryProcessor.executeInternal(String.format(peers_delete_query, SYSTEM_KEYSPACE_NAME, PEERS_V2), addresses.broadcastAddress.getAddress(), addresses.broadcastAddress.getPort());
+            QueryProcessor.executeInternal(String.format(legacy_peers_delete_query, SYSTEM_KEYSPACE_NAME, LEGACY_PEERS), addresses.broadcastAddress.getAddress());
+        }
+        else if (NodeState.isPreJoin(next.directory.peerState(nodeId)))
+        {
+            logger.debug("{} is in pre-join state {}, not updating system.peers_v2 table", nodeId, next.directory.peerState(nodeId));
+        }
+        else
+        {
+            NodeAddresses addresses = next.directory.getNodeAddresses(nodeId);
+            NodeAddresses oldAddresses = prev.directory.getNodeAddresses(nodeId);
+            if (oldAddresses != null && !oldAddresses.equals(addresses))
+            {
+                logger.debug("Purging {} from system.peers_v2 table", oldAddresses);
+                QueryProcessor.executeInternal(String.format(peers_delete_query, SYSTEM_KEYSPACE_NAME, PEERS_V2), oldAddresses.broadcastAddress.getAddress(), oldAddresses.broadcastAddress.getPort());
+                QueryProcessor.executeInternal(String.format(legacy_peers_delete_query, SYSTEM_KEYSPACE_NAME, LEGACY_PEERS), oldAddresses.broadcastAddress.getAddress());
+            }
+
+            Location location = next.directory.location(nodeId);
+
+            Set<String> tokens = SystemKeyspace.tokensAsSet(next.tokenMap.tokens(nodeId));
+            QueryProcessor.executeInternal(String.format(peers_v2_query, SYSTEM_KEYSPACE_NAME, PEERS_V2),
+                                           addresses.broadcastAddress.getAddress(), addresses.broadcastAddress.getPort(),
+                                           addresses.broadcastAddress.getAddress(), addresses.broadcastAddress.getPort(),
+                                           addresses.nativeAddress.getAddress(), addresses.nativeAddress.getPort(),
+                                           location.datacenter, location.rack,
+                                           nodeId.toUUID(),
+                                           next.directory.version(nodeId).cassandraVersion.toString(),
+                                           next.schema.getVersion(),
+                                           tokens);
+
+            QueryProcessor.executeInternal(String.format(legacy_peers_query, SYSTEM_KEYSPACE_NAME, LEGACY_PEERS),
+                                           addresses.broadcastAddress.getAddress(), addresses.broadcastAddress.getAddress(), addresses.nativeAddress.getAddress(),
+                                           location.datacenter, location.rack,
+                                           nodeId.toUUID(),
+                                           next.directory.version(nodeId).cassandraVersion.toString(),
+                                           next.schema.getVersion(),
+                                           tokens);
+        }
+    }
+}

src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java

@@ -52,6 +52,9 @@ public final class SystemViewsKeyspace extends VirtualKeyspace
                     .add(new QueriesTable(VIRTUAL_VIEWS))
                     .add(new LogMessagesTable(VIRTUAL_VIEWS))
                     .add(new SnapshotsTable(VIRTUAL_VIEWS))
+                    .add(new PeersTable(VIRTUAL_VIEWS))
+                    .add(new LocalTable(VIRTUAL_VIEWS))
+                    .add(new ClusterMetadataLogTable(VIRTUAL_VIEWS))
                     .addAll(LocalRepairTables.getAll(VIRTUAL_VIEWS))
                     .addAll(CIDRFilteringMetricsTable.getAll(VIRTUAL_VIEWS))
                     .addAll(StorageAttachedIndexTables.getAll(VIRTUAL_VIEWS))

src/java/org/apache/cassandra/dht/AbstractBounds.java

@@ -50,7 +50,7 @@ public abstract class AbstractBounds<T extends RingPosition<T>> implements Seria
 
     public AbstractBounds(T left, T right)
     {
-        assert left.getPartitioner() == right.getPartitioner();
+        assert left.getPartitioner().getClass().equals(right.getPartitioner().getClass()); // todo: is this enough?
         this.left = left;
         this.right = right;
     }

src/java/org/apache/cassandra/dht/BootStrapper.java

@@ -17,25 +17,33 @@
  */
 package org.apache.cassandra.dht;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.cassandra.tcm.ownership.MovementMap;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.tokenallocator.TokenAllocation;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.streaming.*;
+import org.apache.cassandra.streaming.StreamEvent;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.streaming.StreamEventHandler;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.progress.ProgressEvent;
 import org.apache.cassandra.utils.progress.ProgressEventNotifierSupport;
 import org.apache.cassandra.utils.progress.ProgressEventType;
@@ -47,39 +55,49 @@ public class BootStrapper extends ProgressEventNotifierSupport
     /* endpoint that needs to be bootstrapped */
     protected final InetAddressAndPort address;
     /* token of the node being bootstrapped. */
-    protected final Collection<Token> tokens;
+    protected final ClusterMetadata metadata;
-    protected final TokenMetadata tokenMetadata;
+    private final MovementMap movements;
+    private final MovementMap strictMovements;
 
-    public BootStrapper(InetAddressAndPort address, Collection<Token> tokens, TokenMetadata tmd)
+    public BootStrapper(InetAddressAndPort address,
+                        ClusterMetadata metadata,
+                        MovementMap movements,
+                        MovementMap strictMovements)
     {
         assert address != null;
-        assert tokens != null && !tokens.isEmpty();
 
         this.address = address;
-        this.tokens = tokens;
+        this.metadata = metadata;
-        this.tokenMetadata = tmd;
+        this.movements = movements;
+        this.strictMovements = strictMovements;
     }
 
-    public Future<StreamState> bootstrap(StreamStateStore stateStore, boolean useStrictConsistency)
+    public Future<StreamState> bootstrap(StreamStateStore stateStore, boolean useStrictConsistency, InetAddressAndPort beingReplaced)
     {
         logger.trace("Beginning bootstrap process");
 
-        RangeStreamer streamer = new RangeStreamer(tokenMetadata,
+        RangeStreamer streamer = new RangeStreamer(metadata,
-                                                   tokens,
-                                                   address,
                                                    StreamOperation.BOOTSTRAP,
                                                    useStrictConsistency,
                                                    DatabaseDescriptor.getEndpointSnitch(),
                                                    stateStore,
                                                    true,
-                                                   DatabaseDescriptor.getStreamingConnectionsPerHost());
+                                                   DatabaseDescriptor.getStreamingConnectionsPerHost(),
-        final Collection<String> nonLocalStrategyKeyspaces = Schema.instance.distributedKeyspaces().names();
+                                                   movements,
+                                                   strictMovements);
+
+        if (beingReplaced != null)
+            streamer.addSourceFilter(new RangeStreamer.ExcludedSourcesFilter(Collections.singleton(beingReplaced)));
+
+        final Collection<String> nonLocalStrategyKeyspaces = Schema.instance.getNonLocalStrategyKeyspaces().names();
         if (nonLocalStrategyKeyspaces.isEmpty())
             logger.debug("Schema does not contain any non-local keyspaces to stream on bootstrap");
         for (String keyspaceName : nonLocalStrategyKeyspaces)
         {
-            AbstractReplicationStrategy strategy = Keyspace.open(keyspaceName).getReplicationStrategy();
+            KeyspaceMetadata ksm = metadata.schema.getKeyspaces().get(keyspaceName).get();
-            streamer.addRanges(keyspaceName, strategy.getPendingAddressRanges(tokenMetadata, tokens, address));
+            if (ksm.params.replication.isMeta())
+                continue;
+            streamer.addKeyspaceToFetch(keyspaceName);
         }
 
         StreamResultFuture bootstrapStreamResult = streamer.fetchAsync();
@@ -153,7 +171,7 @@ public class BootStrapper extends ProgressEventNotifierSupport
      * otherwise, if allocationKeyspace is specified use the token allocation algorithm to generate suitable tokens
      * else choose num_tokens tokens at random
      */
-    public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddressAndPort address, long schemaTimeoutMillis, long ringTimeoutMillis) throws ConfigurationException
+    public static Collection<Token> getBootstrapTokens(final ClusterMetadata metadata, InetAddressAndPort address) throws ConfigurationException
     {
         String allocationKeyspace = DatabaseDescriptor.getAllocateTokensForKeyspace();
         Integer allocationLocalRf = DatabaseDescriptor.getAllocateTokensForLocalRf();
@@ -174,10 +192,10 @@ public class BootStrapper extends ProgressEventNotifierSupport
             throw new ConfigurationException("num_tokens must be >= 1");
 
         if (allocationKeyspace != null)
-            return allocateTokens(metadata, address, allocationKeyspace, numTokens, schemaTimeoutMillis, ringTimeoutMillis);
+            return allocateTokens(metadata, address, allocationKeyspace, numTokens);
 
         if (allocationLocalRf != null)
-            return allocateTokens(metadata, address, allocationLocalRf, numTokens, schemaTimeoutMillis, ringTimeoutMillis);
+            return allocateTokens(metadata, address, allocationLocalRf, numTokens);
 
         if (numTokens == 1)
             logger.warn("Picking random token for a single vnode.  You should probably add more vnodes and/or use the automatic token allocation mechanism.");
@@ -187,32 +205,26 @@ public class BootStrapper extends ProgressEventNotifierSupport
         return tokens;
     }
 
-    private static Collection<Token> getSpecifiedTokens(final TokenMetadata metadata,
+    private static Collection<Token> getSpecifiedTokens(final ClusterMetadata metadata,
                                                         Collection<String> initialTokens)
     {
         logger.info("tokens manually specified as {}",  initialTokens);
         List<Token> tokens = new ArrayList<>(initialTokens.size());
         for (String tokenString : initialTokens)
         {
-            Token token = metadata.partitioner.getTokenFactory().fromString(tokenString);
+            Token token = metadata.tokenMap.partitioner().getTokenFactory().fromString(tokenString);
-            if (metadata.getEndpoint(token) != null)
+            if (metadata.tokenMap.owner(token) != null)
                 throw new ConfigurationException("Bootstrapping to existing token " + tokenString + " is not allowed (decommission/removenode the old node first).");
             tokens.add(token);
         }
         return tokens;
     }
 
-    static Collection<Token> allocateTokens(final TokenMetadata metadata,
+    static Collection<Token> allocateTokens(final ClusterMetadata metadata,
                                             InetAddressAndPort address,
                                             String allocationKeyspace,
-                                            int numTokens,
+                                            int numTokens)
-                                            long schemaTimeoutMillis,
-                                            long ringTimeoutMillis)
     {
-        StorageService.instance.waitForSchema(schemaTimeoutMillis, ringTimeoutMillis);
-        if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
-            Gossiper.waitToSettle();
-
         Keyspace ks = Keyspace.open(allocationKeyspace);
         if (ks == null)
             throw new ConfigurationException("Problem opening token allocation keyspace " + allocationKeyspace);
@@ -224,33 +236,35 @@ public class BootStrapper extends ProgressEventNotifierSupport
     }
 
 
-    static Collection<Token> allocateTokens(final TokenMetadata metadata,
+    static Collection<Token> allocateTokens(final ClusterMetadata metadata,
                                             InetAddressAndPort address,
                                             int rf,
-                                            int numTokens,
+                                            int numTokens)
-                                            long schemaTimeoutMillis,
-                                            long ringTimeoutMillis)
     {
-        StorageService.instance.waitForSchema(schemaTimeoutMillis, ringTimeoutMillis);
-        if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
-            Gossiper.waitToSettle();
-
         Collection<Token> tokens = TokenAllocation.allocateTokens(metadata, rf, address, numTokens);
         BootstrapDiagnostics.tokensAllocated(address, metadata, rf, numTokens, tokens);
         return tokens;
     }
 
-    public static Collection<Token> getRandomTokens(TokenMetadata metadata, int numTokens)
+    public static Set<Token> getRandomTokens(ClusterMetadata metadata, int numTokens)
     {
         Set<Token> tokens = new HashSet<>(numTokens);
         while (tokens.size() < numTokens)
         {
-            Token token = metadata.partitioner.getRandomToken();
+            Token token = metadata.tokenMap.partitioner().getRandomToken();
-            if (metadata.getEndpoint(token) == null)
+            if (metadata.tokenMap.owner(token) == null)
                 tokens.add(token);
         }
 
         logger.info("Generated random tokens. tokens are {}", tokens);
         return tokens;
     }
+
+    public String toString()
+    {
+        return "BootStrapper{" +
+               "address=" + address +
+               ", metadata=" + metadata +
+               '}';
+    }
 }

src/java/org/apache/cassandra/dht/BootstrapDiagnostics.java

@@ -19,12 +19,13 @@
 package org.apache.cassandra.dht;
 
 import java.util.Collection;
+
 import com.google.common.collect.ImmutableList;
 
 import org.apache.cassandra.dht.BootstrapEvent.BootstrapEventType;
 import org.apache.cassandra.diag.DiagnosticEventService;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.tcm.ClusterMetadata;
 
 /**
  * Utility methods for bootstrap related activities.
@@ -50,38 +51,38 @@ final class BootstrapDiagnostics
                                                ImmutableList.copyOf(initialTokens)));
     }
 
-    static void useRandomTokens(InetAddressAndPort address, TokenMetadata metadata, int numTokens, Collection<Token> tokens)
+    static void useRandomTokens(InetAddressAndPort address, ClusterMetadata metadata, int numTokens, Collection<Token> tokens)
     {
         if (isEnabled(BootstrapEventType.BOOTSTRAP_USING_RANDOM_TOKENS))
             service.publish(new BootstrapEvent(BootstrapEventType.BOOTSTRAP_USING_RANDOM_TOKENS,
                                                address,
-                                               metadata.cloneOnlyTokenMap(),
+                                               metadata,
                                                null,
                                                null,
                                                numTokens,
                                                ImmutableList.copyOf(tokens)));
     }
 
-    static void tokensAllocated(InetAddressAndPort address, TokenMetadata metadata,
+    static void tokensAllocated(InetAddressAndPort address, ClusterMetadata metadata,
                                 String allocationKeyspace, int numTokens, Collection<Token> tokens)
     {
         if (isEnabled(BootstrapEventType.TOKENS_ALLOCATED))
             service.publish(new BootstrapEvent(BootstrapEventType.TOKENS_ALLOCATED,
                                                address,
-                                               metadata.cloneOnlyTokenMap(),
+                                               metadata,
                                                allocationKeyspace,
                                                null,
                                                numTokens,
                                                ImmutableList.copyOf(tokens)));
     }
 
-    static void tokensAllocated(InetAddressAndPort address, TokenMetadata metadata,
+    static void tokensAllocated(InetAddressAndPort address, ClusterMetadata metadata,
                                 int rf, int numTokens, Collection<Token> tokens)
     {
         if (isEnabled(BootstrapEventType.TOKENS_ALLOCATED))
             service.publish(new BootstrapEvent(BootstrapEventType.TOKENS_ALLOCATED,
                                                address,
-                                               metadata.cloneOnlyTokenMap(),
+                                               metadata,
                                                null,
                                                rf,
                                                numTokens,

src/java/org/apache/cassandra/dht/BootstrapEvent.java

@@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableCollection;
 
 import org.apache.cassandra.diag.DiagnosticEvent;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.tcm.ClusterMetadata;
 
 /**
  * DiagnosticEvent implementation for bootstrap related activities.
@@ -38,7 +38,7 @@ final class BootstrapEvent extends DiagnosticEvent
 
     private final BootstrapEventType type;
     @Nullable
-    private final TokenMetadata tokenMetadata;
+    private final ClusterMetadata metadata;
     private final InetAddressAndPort address;
     @Nullable
     private final String allocationKeyspace;
@@ -47,12 +47,12 @@ final class BootstrapEvent extends DiagnosticEvent
     private final Integer numTokens;
     private final Collection<Token> tokens;
 
-    BootstrapEvent(BootstrapEventType type, InetAddressAndPort address, @Nullable TokenMetadata tokenMetadata,
+    BootstrapEvent(BootstrapEventType type, InetAddressAndPort address, @Nullable ClusterMetadata metadata,
                    @Nullable String allocationKeyspace, @Nullable Integer rf, int numTokens, ImmutableCollection<Token> tokens)
     {
         this.type = type;
         this.address = address;
-        this.tokenMetadata = tokenMetadata;
+        this.metadata = metadata;
         this.allocationKeyspace = allocationKeyspace;
         this.rf = rf;
         this.numTokens = numTokens;
@@ -76,7 +76,7 @@ final class BootstrapEvent extends DiagnosticEvent
     {
         // be extra defensive against nulls and bugs
         HashMap<String, Serializable> ret = new HashMap<>();
-        ret.put("tokenMetadata", String.valueOf(tokenMetadata));
+        ret.put("metadata", metadata.legacyToString());
         ret.put("allocationKeyspace", allocationKeyspace);
         ret.put("rf", rf);
         ret.put("numTokens", numTokens);

src/java/org/apache/cassandra/dht/ComparableObjectToken.java

@@ -62,7 +62,7 @@ abstract class ComparableObjectToken<C extends Comparable<C>> extends Token
     public int compareTo(Token o)
     {
         if (o.getClass() != getClass())
-            throw new IllegalArgumentException("Invalid type of Token.compareTo() argument.");
+            throw new IllegalArgumentException(String.format("Invalid type of Token.compareTo() argument. %s != %s", o.getClass(), getClass()));
 
         return token.compareTo(((ComparableObjectToken<C>) o).token);
     }