Refactor schema management to allow for schema source pluggability

Patch by Jacek Lewandowski, reviewed by Alex Petrov for CASSANDRA-17044.

src/java/org/apache/cassandra/auth/AuthSchemaChangeListener.java

@@ -17,38 +17,43 @@
  */
 package org.apache.cassandra.auth;
 
-import java.util.List;
-
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.SchemaChangeListener;
+import org.apache.cassandra.schema.TableMetadata;
 
 /**
  * SchemaChangeListener implementation that cleans up permissions on dropped resources.
  */
-public class AuthSchemaChangeListener extends SchemaChangeListener
+public class AuthSchemaChangeListener implements SchemaChangeListener
 {
-    public void onDropKeyspace(String ksName)
+    @Override
+    public void onDropKeyspace(KeyspaceMetadata keyspace)
     {
-        DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.keyspace(ksName));
-        DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.allTables(ksName));
-        DatabaseDescriptor.getAuthorizer().revokeAllOn(FunctionResource.keyspace(ksName));
+        DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.keyspace(keyspace.name));
+        DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.allTables(keyspace.name));
+        DatabaseDescriptor.getAuthorizer().revokeAllOn(FunctionResource.keyspace(keyspace.name));
     }
 
-    public void onDropTable(String ksName, String cfName)
+    @Override
+    public void onDropTable(TableMetadata table)
     {
-        DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.table(ksName, cfName));
+        DatabaseDescriptor.getAuthorizer().revokeAllOn(DataResource.table(table.keyspace, table.name));
     }
 
-    public void onDropFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+    @Override
+    public void onDropFunction(UDFunction function)
     {
         DatabaseDescriptor.getAuthorizer()
-                          .revokeAllOn(FunctionResource.function(ksName, functionName, argTypes));
+                          .revokeAllOn(FunctionResource.function(function.name().keyspace, function.name().name, function.argTypes()));
     }
 
-    public void onDropAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+    @Override
+    public void onDropAggregate(UDAggregate aggregate)
     {
         DatabaseDescriptor.getAuthorizer()
-                          .revokeAllOn(FunctionResource.function(ksName, aggregateName, argTypes));
+                          .revokeAllOn(FunctionResource.function(aggregate.name().keyspace, aggregate.name().name, aggregate.argTypes()));
     }
 }

src/java/org/apache/cassandra/batchlog/BatchlogManager.java

@@ -112,7 +112,7 @@ public class BatchlogManager implements BatchlogManagerMBean
         MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
 
         batchlogTasks.scheduleWithFixedDelay(this::replayFailedBatches,
-                                             StorageService.RING_DELAY,
+                                             StorageService.RING_DELAY_MILLIS,
                                              REPLAY_INTERVAL,
                                              MILLISECONDS);
     }

src/java/org/apache/cassandra/cache/AutoSavingCache.java

@@ -347,11 +347,6 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
 
                 //Need to be able to check schema version because CF names are ambiguous
                 UUID schemaVersion = Schema.instance.getVersion();
-                if (schemaVersion == null)
-                {
-                    Schema.instance.updateVersion();
-                    schemaVersion = Schema.instance.getVersion();
-                }
                 writer.writeLong(schemaVersion.getMostSignificantBits());
                 writer.writeLong(schemaVersion.getLeastSignificantBits());
 

src/java/org/apache/cassandra/cql3/QueryProcessor.java

@@ -38,18 +38,22 @@ import org.antlr.runtime.*;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.selection.ResultSetBuilder;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.ClientRequestMetrics;
 import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaChangeListener;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.selection.ResultSetBuilder;
 import org.apache.cassandra.cql3.statements.*;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.rows.Row;
@@ -915,7 +919,7 @@ public class QueryProcessor implements QueryHandler
         preparedStatements.asMap().clear();
     }
 
-    private static class StatementInvalidatingListener extends SchemaChangeListener
+    private static class StatementInvalidatingListener implements SchemaChangeListener
     {
         private static void removeInvalidPreparedStatements(String ksName, String cfName)
         {
@@ -1001,14 +1005,16 @@ public class QueryProcessor implements QueryHandler
             return ksName.equals(statementKsName) && (cfName == null || cfName.equals(statementCfName));
         }
 
-        public void onCreateFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onCreateFunction(UDFunction function)
         {
-            onCreateFunctionInternal(ksName, functionName, argTypes);
+            onCreateFunctionInternal(function.name().keyspace, function.name().name, function.argTypes());
         }
 
-        public void onCreateAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onCreateAggregate(UDAggregate aggregate)
         {
-            onCreateFunctionInternal(ksName, aggregateName, argTypes);
+            onCreateFunctionInternal(aggregate.name().keyspace, aggregate.name().name, aggregate.argTypes());
         }
 
         private static void onCreateFunctionInternal(String ksName, String functionName, List<AbstractType<?>> argTypes)
@@ -1019,51 +1025,58 @@ public class QueryProcessor implements QueryHandler
                 removeInvalidPreparedStatementsForFunction(ksName, functionName);
         }
 
-        public void onAlterTable(String ksName, String cfName, boolean affectsStatements)
+        @Override
+        public void onAlterTable(TableMetadata before, TableMetadata after, boolean affectsStatements)
         {
-            logger.trace("Column definitions for {}.{} changed, invalidating related prepared statements", ksName, cfName);
+            logger.trace("Column definitions for {}.{} changed, invalidating related prepared statements", before.keyspace, before.name);
             if (affectsStatements)
-                removeInvalidPreparedStatements(ksName, cfName);
+                removeInvalidPreparedStatements(before.keyspace, before.name);
         }
 
-        public void onAlterFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onAlterFunction(UDFunction before, UDFunction after)
         {
             // Updating a function may imply we've changed the body of the function, so we need to invalid statements so that
             // the new definition is picked (the function is resolved at preparation time).
             // TODO: if the function has multiple overload, we could invalidate only the statement refering to the overload
             // that was updated. This requires a few changes however and probably doesn't matter much in practice.
-            removeInvalidPreparedStatementsForFunction(ksName, functionName);
+            removeInvalidPreparedStatementsForFunction(before.name().keyspace, before.name().name);
         }
 
-        public void onAlterAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onAlterAggregate(UDAggregate before, UDAggregate after)
         {
             // Updating a function may imply we've changed the body of the function, so we need to invalid statements so that
             // the new definition is picked (the function is resolved at preparation time).
             // TODO: if the function has multiple overload, we could invalidate only the statement refering to the overload
             // that was updated. This requires a few changes however and probably doesn't matter much in practice.
-            removeInvalidPreparedStatementsForFunction(ksName, aggregateName);
+            removeInvalidPreparedStatementsForFunction(before.name().keyspace, before.name().name);
         }
 
-        public void onDropKeyspace(String ksName)
+        @Override
+        public void onDropKeyspace(KeyspaceMetadata keyspace)
         {
-            logger.trace("Keyspace {} was dropped, invalidating related prepared statements", ksName);
-            removeInvalidPreparedStatements(ksName, null);
+            logger.trace("Keyspace {} was dropped, invalidating related prepared statements", keyspace.name);
+            removeInvalidPreparedStatements(keyspace.name, null);
         }
 
-        public void onDropTable(String ksName, String cfName)
+        @Override
+        public void onDropTable(TableMetadata table)
         {
-            logger.trace("Table {}.{} was dropped, invalidating related prepared statements", ksName, cfName);
-            removeInvalidPreparedStatements(ksName, cfName);
+            logger.trace("Table {}.{} was dropped, invalidating related prepared statements", table.keyspace, table.name);
+            removeInvalidPreparedStatements(table.keyspace, table.name);
         }
 
-        public void onDropFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onDropFunction(UDFunction function)
         {
-            removeInvalidPreparedStatementsForFunction(ksName, functionName);
+            removeInvalidPreparedStatementsForFunction(function.name().keyspace, function.name().name);
         }
 
-        public void onDropAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onDropAggregate(UDAggregate aggregate)
         {
-            removeInvalidPreparedStatementsForFunction(ksName, aggregateName);
+            removeInvalidPreparedStatementsForFunction(aggregate.name().keyspace, aggregate.name().name);
         }
     }
 }

src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java

@@ -132,7 +132,7 @@ public abstract class DescribeStatement<T> extends CQLStatement.Raw implements C
     @Override
     public ResultMessage executeLocally(QueryState state, QueryOptions options)
     {
-        Keyspaces keyspaces = Schema.instance.snapshot();
+        Keyspaces keyspaces = Schema.instance.distributedAndLocalKeyspaces();
         UUID schemaVersion = Schema.instance.getVersion();
 
         keyspaces = Keyspaces.builder()

src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java

@@ -105,11 +105,11 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
 
         validateKeyspaceName();
 
-        KeyspacesDiff diff = MigrationManager.announce(this, locally);
+        SchemaTransformationResult result = Schema.instance.transform(this, locally);
 
-        clientWarnings(diff).forEach(ClientWarn.instance::warn);
+        clientWarnings(result.diff).forEach(ClientWarn.instance::warn);
 
-        if (diff.isEmpty())
+        if (result.diff.isEmpty())
             return new ResultMessage.Void();
 
         /*
@@ -121,9 +121,9 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
          */
         AuthenticatedUser user = state.getClientState().getUser();
         if (null != user && !user.isAnonymous())
-            createdResources(diff).forEach(r -> grantPermissionsOnResource(r, user));
+            createdResources(result.diff).forEach(r -> grantPermissionsOnResource(r, user));
 
-        return new ResultMessage.SchemaChange(schemaChangeEvent(diff));
+        return new ResultMessage.SchemaChange(schemaChangeEvent(result.diff));
     }
 
     private void validateKeyspaceName()

src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java

@@ -85,7 +85,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         this.tableName = tableName;
     }
 
-    public Keyspaces apply(Keyspaces schema) throws UnknownHostException
+    public Keyspaces apply(Keyspaces schema)
     {
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
 
@@ -123,7 +123,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
         return format("%s (%s, %s)", getClass().getSimpleName(), keyspaceName, tableName);
     }
 
-    abstract KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table) throws UnknownHostException;
+    abstract KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table);
 
     /**
      * ALTER TABLE <table> ALTER <column> TYPE <newtype>;

src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java

@@ -140,6 +140,7 @@ public final class CreateTableStatement extends AlterSchemaStatement
             {
                 int totalUserTables = Schema.instance.getUserKeyspaces()
                                                      .stream()
+                                                     .map(ksm -> ksm.name)
                                                      .map(Keyspace::open)
                                                      .mapToInt(keyspace -> keyspace.getColumnFamilyStores().size())
                                                      .sum();

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -575,10 +575,15 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     /** call when dropping or renaming a CF. Performs mbean housekeeping and invalidates CFS to other operations */
     public void invalidate()
     {
-        invalidate(true);
+        invalidate(true, true);
     }
 
     public void invalidate(boolean expectMBean)
+    {
+        invalidate(expectMBean, true);
+    }
+
+    public void invalidate(boolean expectMBean, boolean dropData)
     {
         // disable and cancel in-progress compactions before invalidating
         valid = false;
@@ -600,9 +605,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         compactionStrategyManager.shutdown();
         SystemKeyspace.removeTruncationRecord(metadata.id);
 
-        data.dropSSTables();
-        LifecycleTransaction.waitForDeletions();
-        indexManager.dropAllIndexes();
+        if (dropData)
+        {
+            data.dropSSTables();
+            LifecycleTransaction.waitForDeletions();
+        }
+        indexManager.dropAllIndexes(dropData);
 
         invalidateCaches();
     }
@@ -2021,7 +2029,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             try (PrintStream out = new PrintStream(new FileOutputStreamPlus(schemaFile)))
             {
                 SchemaCQLHelper.reCreateStatementsForSchemaCql(metadata(),
-                                                               keyspace.getMetadata().types)
+                                                               keyspace.getMetadata())
                                .forEach(out::println);
             }
         }

src/java/org/apache/cassandra/db/Keyspace.java

@@ -129,9 +129,20 @@ public class Keyspace
         initialized = true;
     }
 
+    /**
+     * Never use it in production code.
+     *
+     * Useful when creating a fake Schema so that it does not manage Keyspace instances (and CFS)
+     */
+    @VisibleForTesting
+    public static void unsetInitialized()
+    {
+        initialized = false;
+    }
+
     public static Keyspace open(String keyspaceName)
     {
-        assert initialized || SchemaConstants.isLocalSystemKeyspace(keyspaceName);
+        assert initialized || SchemaConstants.isLocalSystemKeyspace(keyspaceName) : "Initialized: " + initialized;
         return open(keyspaceName, Schema.instance, true);
     }
 
@@ -141,8 +152,7 @@ public class Keyspace
         return open(keyspaceName, Schema.instance, false);
     }
 
-    @VisibleForTesting
-    static Keyspace open(String keyspaceName, SchemaProvider schema, boolean loadSSTables)
+    public static Keyspace open(String keyspaceName, SchemaProvider schema, boolean loadSSTables)
     {
         return schema.maybeAddKeyspaceInstance(keyspaceName, () -> new Keyspace(keyspaceName, schema, loadSSTables));
     }
@@ -372,33 +382,32 @@ public class Keyspace
         replicationParams = ksm.params.replication;
     }
 
-    // best invoked on the compaction mananger.
-    public void dropCf(TableId tableId)
+    // best invoked on the compaction manager.
+    public void dropCf(TableId tableId, boolean dropData)
     {
-        assert columnFamilyStores.containsKey(tableId);
         ColumnFamilyStore cfs = columnFamilyStores.remove(tableId);
         if (cfs == null)
             return;
 
         cfs.onTableDropped();
-        unloadCf(cfs);
+        unloadCf(cfs, dropData);
     }
 
     /**
      * Unloads all column family stores and releases metrics.
      */
-    public void unload()
+    public void unload(boolean dropData)
     {
         for (ColumnFamilyStore cfs : getColumnFamilyStores())
-            unloadCf(cfs);
+            unloadCf(cfs, dropData);
         metric.release();
     }
 
     // disassociate a cfs from this keyspace instance.
-    private void unloadCf(ColumnFamilyStore cfs)
+    private void unloadCf(ColumnFamilyStore cfs, boolean dropData)
     {
         cfs.forceBlockingFlush();
-        cfs.invalidate();
+        cfs.invalidate(true, dropData);
     }
 
     /**
@@ -755,12 +764,12 @@ public class Keyspace
 
     public static Iterable<Keyspace> nonSystem()
     {
-        return Iterables.transform(Schema.instance.getNonSystemKeyspaces(), Keyspace::open);
+        return Iterables.transform(Schema.instance.getNonSystemKeyspaces().names(), Keyspace::open);
     }
 
     public static Iterable<Keyspace> nonLocalStrategy()
     {
-        return Iterables.transform(Schema.instance.getNonLocalStrategyKeyspaces(), Keyspace::open);
+        return Iterables.transform(Schema.instance.getNonLocalStrategyKeyspaces().names(), Keyspace::open);
     }
 
     public static Iterable<Keyspace> system()

src/java/org/apache/cassandra/db/SchemaCQLHelper.java

@@ -42,12 +42,12 @@ public class SchemaCQLHelper
     /**
      * Generates the DDL statement for a {@code schema.cql} snapshot file.
      */
-    public static Stream<String> reCreateStatementsForSchemaCql(TableMetadata metadata, Types types)
+    public static Stream<String> reCreateStatementsForSchemaCql(TableMetadata metadata, KeyspaceMetadata keyspaceMetadata)
     {
         // Types come first, as table can't be created without them
-        Stream<String> udts = SchemaCQLHelper.getUserTypesAsCQL(metadata, types, true);
+        Stream<String> udts = SchemaCQLHelper.getUserTypesAsCQL(metadata, keyspaceMetadata.types, true);
 
-        Stream<String> tableMatadata = Stream.of(SchemaCQLHelper.getTableMetadataAsCQL(metadata));
+        Stream<String> tableMatadata = Stream.of(SchemaCQLHelper.getTableMetadataAsCQL(metadata, keyspaceMetadata));
 
         Stream<String> indexes = SchemaCQLHelper.getIndexesAsCQL(metadata, true);
         return Stream.of(udts, tableMatadata, indexes).flatMap(Function.identity());
@@ -60,11 +60,10 @@ public class SchemaCQLHelper
      * that will not contain everything needed for user types.
      */
     @VisibleForTesting
-    public static String getTableMetadataAsCQL(TableMetadata metadata)
+    public static String getTableMetadataAsCQL(TableMetadata metadata, KeyspaceMetadata keyspaceMetadata)
     {
         if (metadata.isView())
         {
-            KeyspaceMetadata keyspaceMetadata = Schema.instance.getKeyspaceMetadata(metadata.keyspace);
             ViewMetadata viewMetadata = keyspaceMetadata.views.get(metadata.name).orElse(null);
             assert viewMetadata != null;
             /*

src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaChangeListener;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
@@ -48,7 +49,7 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
  *
  * See CASSANDRA-7688.
  */
-public class SizeEstimatesRecorder extends SchemaChangeListener implements Runnable
+public class SizeEstimatesRecorder implements SchemaChangeListener, Runnable
 {
     private static final Logger logger = LoggerFactory.getLogger(SizeEstimatesRecorder.class);
 
@@ -177,8 +178,8 @@ public class SizeEstimatesRecorder extends SchemaChangeListener implements Runna
     }
 
     @Override
-    public void onDropTable(String keyspace, String table)
+    public void onDropTable(TableMetadata table)
     {
-        SystemKeyspace.clearEstimates(keyspace, table);
+        SystemKeyspace.clearEstimates(table.keyspace, table.name);
     }
 }

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -491,11 +491,6 @@ public final class SystemKeyspace
         DECOMMISSIONED
     }
 
-    public static void finishStartup()
-    {
-        Schema.instance.saveSystemKeyspace();
-    }
-
     public static void persistLocalMetadata()
     {
         String req = "INSERT INTO system.%s (" +
@@ -1174,6 +1169,20 @@ public final class SystemKeyspace
         return hostId;
     }
 
+    /**
+     * Gets the schema version or null if missing
+     */
+    public static UUID getSchemaVersion()
+    {
+        String req = "SELECT schema_version FROM system.%s WHERE key='%s'";
+        UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
+
+        if (!result.isEmpty() && result.one().has("schema_version"))
+            return result.one().getUUID("schema_version");
+
+        return null;
+    }
+
     /**
      * Gets the stored rack for the local node, or null if none have been set yet.
      */

src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java

@@ -202,7 +202,7 @@ public abstract class AbstractCommitLogSegmentManager
                 if (flushingSize + unused >= 0)
                     break;
             }
-            flushDataFrom(segmentsToRecycle, false);
+            flushDataFrom(segmentsToRecycle, Collections.emptyList(), false);
         }
     }
 
@@ -294,7 +294,7 @@ public abstract class AbstractCommitLogSegmentManager
      * This is necessary to avoid resurrecting data during replay if a user creates a new table with
      * the same name and ID. See CASSANDRA-16986 for more details.
      */
-    void forceRecycleAll(Iterable<TableId> droppedTables)
+    void forceRecycleAll(Collection<TableId> droppedTables)
     {
         List<CommitLogSegment> segmentsToRecycle = new ArrayList<>(activeSegments);
         CommitLogSegment last = segmentsToRecycle.get(segmentsToRecycle.size() - 1);
@@ -308,7 +308,7 @@ public abstract class AbstractCommitLogSegmentManager
         Keyspace.writeOrder.awaitNewBarrier();
 
         // flush and wait for all CFs that are dirty in segments up-to and including 'last'
-        Future<?> future = flushDataFrom(segmentsToRecycle, true);
+        Future<?> future = flushDataFrom(segmentsToRecycle, droppedTables, true);
         try
         {
             future.get();
@@ -394,7 +394,7 @@ public abstract class AbstractCommitLogSegmentManager
      *
      * @return a Future that will finish when all the flushes are complete.
      */
-    private Future<?> flushDataFrom(List<CommitLogSegment> segments, boolean force)
+    private Future<?> flushDataFrom(List<CommitLogSegment> segments, Collection<TableId> droppedTables, boolean force)
     {
         if (segments.isEmpty())
             return ImmediateFuture.success(null);
@@ -407,7 +407,9 @@ public abstract class AbstractCommitLogSegmentManager
         {
             for (TableId dirtyTableId : segment.getDirtyTableIds())
             {
-                TableMetadata metadata = Schema.instance.getTableMetadata(dirtyTableId);
+                TableMetadata metadata = droppedTables.contains(dirtyTableId)
+                                         ? null
+                                         : Schema.instance.getTableMetadata(dirtyTableId);
                 if (metadata == null)
                 {
                     // even though we remove the schema entry before a final flush when dropping a CF,

src/java/org/apache/cassandra/db/commitlog/CommitLog.java

@@ -233,7 +233,7 @@ public class CommitLog implements CommitLogMBean
     /**
      * Flushes all dirty CFs, waiting for them to free and recycle any segments they were retaining
      */
-    public void forceRecycleAllSegments(Iterable<TableId> droppedTables)
+    public void forceRecycleAllSegments(Collection<TableId> droppedTables)
     {
         segmentManager.forceRecycleAll(droppedTables);
     }

src/java/org/apache/cassandra/db/compaction/CompactionManager.java

@@ -1137,7 +1137,7 @@ public class CompactionManager implements CompactionManagerMBean
     /* Used in tests. */
     public void disableAutoCompaction()
     {
-        for (String ksname : Schema.instance.getNonSystemKeyspaces())
+        for (String ksname : Schema.instance.getNonSystemKeyspaces().names())
         {
             for (ColumnFamilyStore cfs : Keyspace.open(ksname).getColumnFamilyStores())
                 cfs.disableAutoCompaction();

src/java/org/apache/cassandra/db/guardrails/Guardrails.java

@@ -175,7 +175,7 @@ public final class Guardrails implements GuardrailsMBean
      */
     public static boolean enabled(ClientState state)
     {
-        return CONFIG_PROVIDER.getOrCreate(state).getEnabled() && DatabaseDescriptor.isDaemonInitialized();
+        return DatabaseDescriptor.isDaemonInitialized() && CONFIG_PROVIDER.getOrCreate(state).getEnabled();
     }
 
     @Override

src/java/org/apache/cassandra/dht/BootStrapper.java

@@ -73,7 +73,7 @@ public class BootStrapper extends ProgressEventNotifierSupport
                                                    stateStore,
                                                    true,
                                                    DatabaseDescriptor.getStreamingConnectionsPerHost());
-        final List<String> nonLocalStrategyKeyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
+        final Collection<String> nonLocalStrategyKeyspaces = Schema.instance.getNonLocalStrategyKeyspaces().names();
         if (nonLocalStrategyKeyspaces.isEmpty())
             logger.debug("Schema does not contain any non-local keyspaces to stream on bootstrap");
         for (String keyspaceName : nonLocalStrategyKeyspaces)
@@ -153,7 +153,7 @@ public class BootStrapper extends ProgressEventNotifierSupport
      * otherwise, if allocationKeyspace is specified use the token allocation algorithm to generate suitable tokens
      * else choose num_tokens tokens at random
      */
-    public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddressAndPort address, long schemaWaitDelay) throws ConfigurationException
+    public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddressAndPort address, long schemaTimeoutMillis, long ringTimeoutMillis) throws ConfigurationException
     {
         String allocationKeyspace = DatabaseDescriptor.getAllocateTokensForKeyspace();
         Integer allocationLocalRf = DatabaseDescriptor.getAllocateTokensForLocalRf();
@@ -174,10 +174,10 @@ public class BootStrapper extends ProgressEventNotifierSupport
             throw new ConfigurationException("num_tokens must be >= 1");
 
         if (allocationKeyspace != null)
-            return allocateTokens(metadata, address, allocationKeyspace, numTokens, schemaWaitDelay);
+            return allocateTokens(metadata, address, allocationKeyspace, numTokens, schemaTimeoutMillis, ringTimeoutMillis);
 
         if (allocationLocalRf != null)
-            return allocateTokens(metadata, address, allocationLocalRf, numTokens, schemaWaitDelay);
+            return allocateTokens(metadata, address, allocationLocalRf, numTokens, schemaTimeoutMillis, ringTimeoutMillis);
 
         if (numTokens == 1)
             logger.warn("Picking random token for a single vnode.  You should probably add more vnodes and/or use the automatic token allocation mechanism.");
@@ -206,9 +206,10 @@ public class BootStrapper extends ProgressEventNotifierSupport
                                             InetAddressAndPort address,
                                             String allocationKeyspace,
                                             int numTokens,
-                                            long schemaWaitDelay)
+                                            long schemaTimeoutMillis,
+                                            long ringTimeoutMillis)
     {
-        StorageService.instance.waitForSchema(schemaWaitDelay);
+        StorageService.instance.waitForSchema(schemaTimeoutMillis, ringTimeoutMillis);
         if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
             Gossiper.waitToSettle();
 
@@ -227,9 +228,10 @@ public class BootStrapper extends ProgressEventNotifierSupport
                                             InetAddressAndPort address,
                                             int rf,
                                             int numTokens,
-                                            long schemaWaitDelay)
+                                            long schemaTimeoutMillis,
+                                            long ringTimeoutMillis)
     {
-        StorageService.instance.waitForSchema(schemaWaitDelay);
+        StorageService.instance.waitForSchema(schemaTimeoutMillis, ringTimeoutMillis);
         if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
             Gossiper.waitToSettle();
 

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -129,7 +129,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     private volatile ScheduledFuture<?> scheduledGossipTask;
     private static final ReentrantLock taskLock = new ReentrantLock();
     public final static int intervalInMillis = 1000;
-    public final static int QUARANTINE_DELAY = GOSSIPER_QUARANTINE_DELAY.getInt(StorageService.RING_DELAY * 2);
+    public final static int QUARANTINE_DELAY = GOSSIPER_QUARANTINE_DELAY.getInt(StorageService.RING_DELAY_MILLIS * 2);
     private static final Logger logger = LoggerFactory.getLogger(Gossiper.class);
     private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 15L, TimeUnit.MINUTES);
 
@@ -780,8 +780,8 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         // remember this node's generation
         int generation = epState.getHeartBeatState().getGeneration();
         logger.info("Removing host: {}", hostId);
-        logger.info("Sleeping for {}ms to ensure {} does not change", StorageService.RING_DELAY, endpoint);
-        Uninterruptibles.sleepUninterruptibly(StorageService.RING_DELAY, TimeUnit.MILLISECONDS);
+        logger.info("Sleeping for {}ms to ensure {} does not change", StorageService.RING_DELAY_MILLIS, endpoint);
+        Uninterruptibles.sleepUninterruptibly(StorageService.RING_DELAY_MILLIS, TimeUnit.MILLISECONDS);
         // make sure it did not change
         epState = endpointStateMap.get(endpoint);
         if (epState.getHeartBeatState().getGeneration() != generation)
@@ -849,8 +849,8 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
             {
                 int generation = epState.getHeartBeatState().getGeneration();
                 int heartbeat = epState.getHeartBeatState().getHeartBeatVersion();
-                logger.info("Sleeping for {}ms to ensure {} does not change", StorageService.RING_DELAY, endpoint);
-                Uninterruptibles.sleepUninterruptibly(StorageService.RING_DELAY, TimeUnit.MILLISECONDS);
+                logger.info("Sleeping for {}ms to ensure {} does not change", StorageService.RING_DELAY_MILLIS, endpoint);
+                Uninterruptibles.sleepUninterruptibly(StorageService.RING_DELAY_MILLIS, TimeUnit.MILLISECONDS);
                 // make sure it did not change
                 EndpointState newState = endpointStateMap.get(endpoint);
                 if (newState == null)
@@ -1652,7 +1652,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     // notify that an application state has changed
-    private void doOnChangeNotifications(InetAddressAndPort addr, ApplicationState state, VersionedValue value)
+    public void doOnChangeNotifications(InetAddressAndPort addr, ApplicationState state, VersionedValue value)
     {
         for (IEndpointStateChangeSubscriber subscriber : subscribers)
         {
@@ -1864,7 +1864,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         boolean isSeed = DatabaseDescriptor.getSeeds().contains(getBroadcastAddressAndPort());
         // We double RING_DELAY if we're not a seed to increase chance of successful startup during a full cluster bounce,
         // giving the seeds a chance to startup before we fail the shadow round
-        int shadowRoundDelay =  isSeed ? StorageService.RING_DELAY : StorageService.RING_DELAY * 2;
+        int shadowRoundDelay = isSeed ? StorageService.RING_DELAY_MILLIS : StorageService.RING_DELAY_MILLIS * 2;
         seedsInShadowRound.clear();
         endpointShadowStateMap.clear();
         // send a completely empty syn

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -780,10 +780,11 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     /**
      * Remove all indexes
      */
-    public void dropAllIndexes()
+    public void dropAllIndexes(boolean dropData)
     {
         markAllIndexesRemoved();
-        invalidateAllIndexesBlocking();
+        if (dropData)
+            invalidateAllIndexesBlocking();
     }
 
     @VisibleForTesting

src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java

@@ -29,6 +29,9 @@ import java.util.SortedSet;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
@@ -42,7 +45,6 @@ import org.apache.cassandra.cql3.functions.types.UserType;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
 import org.apache.cassandra.cql3.statements.UpdateStatement;
 import org.apache.cassandra.db.Clustering;
-import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -101,6 +103,7 @@ public class CQLSSTableWriter implements Closeable
 
     static
     {
+        System.setProperty("cassandra.schema.force_load_local_keyspaces", "true");
         DatabaseDescriptor.clientInitialization(false);
         // Partitioner is not set in client mode.
         if (DatabaseDescriptor.getPartitioner() == null)
@@ -514,24 +517,19 @@ public class CQLSSTableWriter implements Closeable
             if (insertStatement == null)
                 throw new IllegalStateException("No insert statement specified, you should provide an insert statement through using()");
 
+            Preconditions.checkState(Sets.difference(SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES, Schema.instance.getKeyspaces()).isEmpty(),
+                                     "Local keyspaces were not loaded. If this is running as a client, please make sure to add %s=true system property.", Schema.FORCE_LOAD_LOCAL_KEYSPACES_PROP);
             synchronized (CQLSSTableWriter.class)
             {
-                if (Schema.instance.getKeyspaceMetadata(SchemaConstants.SCHEMA_KEYSPACE_NAME) == null)
-                    Schema.instance.load(Schema.getSystemKeyspaceMetadata());
-                if (Schema.instance.getKeyspaceMetadata(SchemaConstants.SYSTEM_KEYSPACE_NAME) == null)
-                    Schema.instance.load(SystemKeyspace.metadata());
 
                 String keyspaceName = schemaStatement.keyspace();
 
-                if (Schema.instance.getKeyspaceMetadata(keyspaceName) == null)
-                {
-                    Schema.instance.load(KeyspaceMetadata.create(keyspaceName,
-                                                                 KeyspaceParams.simple(1),
-                                                                 Tables.none(),
-                                                                 Views.none(),
-                                                                 Types.none(),
-                                                                 Functions.none()));
-                }
+                Schema.instance.transform(SchemaTransformations.addKeyspace(KeyspaceMetadata.create(keyspaceName,
+                                                                                                           KeyspaceParams.simple(1),
+                                                                                                           Tables.none(),
+                                                                                                           Views.none(),
+                                                                                                           Types.none(),
+                                                                                                           Functions.none()), true));
 
                 KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName);
 
@@ -539,8 +537,9 @@ public class CQLSSTableWriter implements Closeable
                 if (tableMetadata == null)
                 {
                     Types types = createTypes(keyspaceName);
+                    Schema.instance.transform(SchemaTransformations.addTypes(types, true));
                     tableMetadata = createTable(types);
-                    Schema.instance.load(ksm.withSwapped(ksm.tables.with(tableMetadata)).withSwapped(types));
+                    Schema.instance.transform(SchemaTransformations.addTable(tableMetadata, true));
                 }
 
                 UpdateStatement preparedInsert = prepareInsert();

src/java/org/apache/cassandra/metrics/TableMetrics.java

@@ -279,7 +279,7 @@ public class TableMetrics
     {
         long total = 0;
         long filtered = 0;
-        for (String keyspace : Schema.instance.getNonSystemKeyspaces())
+        for (String keyspace : Schema.instance.getNonSystemKeyspaces().names())
         {
 
             Keyspace k = Schema.instance.getKeyspaceInstance(keyspace);

src/java/org/apache/cassandra/net/Message.java

@@ -257,6 +257,18 @@ public class Message<T>
         return outWithParam(0, verb, payload, null, null);
     }
 
+    /**
+     * Used by the {@code MultiRangeReadCommand} to split multi-range responses from a replica
+     * into single-range responses.
+     */
+    public static <T> Message<T> remoteResponse(InetAddressAndPort from, Verb verb, T payload)
+    {
+        assert verb.isResponse();
+        long createdAtNanos = approxTime.now();
+        long expiresAtNanos = verb.expiresAtNanos(createdAtNanos);
+        return new Message<>(new Header(0, verb, from, createdAtNanos, expiresAtNanos, 0, NO_PARAMS), payload);
+    }
+
     /** Builds a response Message with provided payload, and all the right fields inferred from request Message */
     public <T> Message<T> responseWith(T payload)
     {

src/java/org/apache/cassandra/net/MessagingService.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.metrics.MessagingMetrics;
 import org.apache.cassandra.service.AbstractWriteResponseHandler;
 import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.FBUtilities;
@@ -198,7 +199,7 @@ import static org.apache.cassandra.utils.Throwables.maybeFail;
  * implemented in {@link org.apache.cassandra.db.virtual.InternodeInboundTable} and
  * {@link org.apache.cassandra.db.virtual.InternodeOutboundTable} respectively.
  */
-public final class MessagingService extends MessagingServiceMBeanImpl
+public class MessagingService extends MessagingServiceMBeanImpl
 {
     private static final Logger logger = LoggerFactory.getLogger(MessagingService.class);
 
@@ -263,7 +264,13 @@ public final class MessagingService extends MessagingServiceMBeanImpl
     @VisibleForTesting
     MessagingService(boolean testOnly)
     {
-        super(testOnly);
+        this(testOnly, new EndpointMessagingVersions(), new MessagingMetrics());
+    }
+
+    @VisibleForTesting
+    MessagingService(boolean testOnly, EndpointMessagingVersions versions, MessagingMetrics metrics)
+    {
+        super(testOnly, versions, metrics);
         OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS);
     }
 

src/java/org/apache/cassandra/net/MessagingServiceMBeanImpl.java

@@ -41,11 +41,13 @@ public class MessagingServiceMBeanImpl implements MessagingServiceMBean
     public final ConcurrentMap<InetAddressAndPort, OutboundConnections> channelManagers = new ConcurrentHashMap<>();
     public final ConcurrentMap<InetAddressAndPort, InboundMessageHandlers> messageHandlers = new ConcurrentHashMap<>();
 
-    public final EndpointMessagingVersions versions = new EndpointMessagingVersions();
-    public final MessagingMetrics metrics = new MessagingMetrics();
+    public final EndpointMessagingVersions versions;
+    public final MessagingMetrics metrics;
 
-    MessagingServiceMBeanImpl(boolean testOnly)
+    public MessagingServiceMBeanImpl(boolean testOnly, EndpointMessagingVersions versions, MessagingMetrics metrics)
     {
+        this.versions = versions;
+        this.metrics = metrics;
         if (!testOnly)
         {
             MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);

src/java/org/apache/cassandra/net/Verb.java

@@ -71,6 +71,7 @@ import org.apache.cassandra.repair.messages.SyncResponse;
 import org.apache.cassandra.repair.messages.SyncRequest;
 import org.apache.cassandra.repair.messages.ValidationResponse;
 import org.apache.cassandra.repair.messages.ValidationRequest;
+import org.apache.cassandra.schema.SchemaMutationsSerializer;
 import org.apache.cassandra.schema.SchemaPullVerbHandler;
 import org.apache.cassandra.schema.SchemaPushVerbHandler;
 import org.apache.cassandra.schema.SchemaVersionVerbHandler;
@@ -102,7 +103,6 @@ import static org.apache.cassandra.concurrent.Stage.*;
 import static org.apache.cassandra.net.VerbTimeouts.*;
 import static org.apache.cassandra.net.Verb.Kind.*;
 import static org.apache.cassandra.net.Verb.Priority.*;
-import static org.apache.cassandra.schema.MigrationManager.MigrationsSerializer;
 
 /**
  * Note that priorities except P0 are presently unused.  P0 corresponds to urgent, i.e. what used to be the "Gossip" connection.
@@ -150,8 +150,8 @@ public enum Verb
 
     // P1 because messages can be arbitrarily large or aren't crucial
     SCHEMA_PUSH_RSP        (98,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    SCHEMA_PUSH_REQ        (18,  P1, rpcTimeout,      MIGRATION,         () -> MigrationsSerializer.instance,        () -> SchemaPushVerbHandler.instance,      SCHEMA_PUSH_RSP     ),
-    SCHEMA_PULL_RSP        (88,  P1, rpcTimeout,      MIGRATION,         () -> MigrationsSerializer.instance,        () -> ResponseVerbHandler.instance                             ),
+    SCHEMA_PUSH_REQ        (18,  P1, rpcTimeout,      MIGRATION,         () -> SchemaMutationsSerializer.instance,   () -> SchemaPushVerbHandler.instance,      SCHEMA_PUSH_RSP     ),
+    SCHEMA_PULL_RSP        (88,  P1, rpcTimeout,      MIGRATION,         () -> SchemaMutationsSerializer.instance,   () -> ResponseVerbHandler.instance                             ),
     SCHEMA_PULL_REQ        (28,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaPullVerbHandler.instance,      SCHEMA_PULL_RSP     ),
     SCHEMA_VERSION_RSP     (80,  P1, rpcTimeout,      MIGRATION,         () -> UUIDSerializer.serializer,            () -> ResponseVerbHandler.instance                             ),
     SCHEMA_VERSION_REQ     (20,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaVersionVerbHandler.instance,   SCHEMA_VERSION_RSP  ),

src/java/org/apache/cassandra/repair/consistent/admin/SchemaArgsParser.java

@@ -73,7 +73,7 @@ public class SchemaArgsParser implements Iterable<ColumnFamilyStore>
         if (schemaArgs.isEmpty())
         {
             // iterate over everything
-            Iterator<String> ksNames = Schema.instance.getNonLocalStrategyKeyspaces().iterator();
+            Iterator<String> ksNames = Schema.instance.getNonLocalStrategyKeyspaces().names().iterator();
 
             return new AbstractIterator<ColumnFamilyStore>()
             {

src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandler.java

@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
+
+import static org.apache.cassandra.schema.MigrationCoordinator.MAX_OUTSTANDING_VERSION_REQUESTS;
+
+public class DefaultSchemaUpdateHandler implements SchemaUpdateHandler, IEndpointStateChangeSubscriber
+{
+    private static final Logger logger = LoggerFactory.getLogger(DefaultSchemaUpdateHandler.class);
+
+    @VisibleForTesting
+    final MigrationCoordinator migrationCoordinator;
+
+    private final boolean requireSchemas;
+    private final BiConsumer<SchemaTransformationResult, Boolean> updateCallback;
+    private volatile DistributedSchema schema = DistributedSchema.EMPTY;
+
+    private MigrationCoordinator createMigrationCoordinator(MessagingService messagingService)
+    {
+        return new MigrationCoordinator(messagingService,
+                                        Stage.MIGRATION.executor(),
+                                        ScheduledExecutors.scheduledTasks,
+                                        MAX_OUTSTANDING_VERSION_REQUESTS,
+                                        Gossiper.instance,
+                                        () -> schema.getVersion(),
+                                        (from, mutations) -> applyMutations(mutations));
+    }
+
+    public DefaultSchemaUpdateHandler(BiConsumer<SchemaTransformationResult, Boolean> updateCallback)
+    {
+        this(null, MessagingService.instance(), !CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK.getBoolean(), updateCallback);
+    }
+
+    public DefaultSchemaUpdateHandler(MigrationCoordinator migrationCoordinator,
+                                      MessagingService messagingService,
+                                      boolean requireSchemas,
+                                      BiConsumer<SchemaTransformationResult, Boolean> updateCallback)
+    {
+        this.requireSchemas = requireSchemas;
+        this.updateCallback = updateCallback;
+        this.migrationCoordinator = migrationCoordinator == null ? createMigrationCoordinator(messagingService) : migrationCoordinator;
+        Gossiper.instance.register(this);
+        SchemaPushVerbHandler.instance.register(msg -> applyMutations(msg.payload));
+        SchemaPullVerbHandler.instance.register(msg -> messagingService.send(msg.responseWith(getSchemaMutations()), msg.from()));
+    }
+
+    public synchronized void start()
+    {
+        if (StorageService.instance.isReplacing())
+            onRemove(DatabaseDescriptor.getReplaceAddress());
+
+        SchemaKeyspace.saveSystemKeyspacesSchema();
+
+        migrationCoordinator.start();
+    }
+
+    @Override
+    public boolean waitUntilReady(Duration timeout)
+    {
+        logger.debug("Waiting for schema to be ready (max {})", timeout);
+        boolean schemasReceived = migrationCoordinator.awaitSchemaRequests(timeout.toMillis());
+
+        if (schemasReceived)
+            return true;
+
+        logger.warn("There are nodes in the cluster with a different schema version than us, from which we did not merge schemas: " +
+                    "our version: ({}), outstanding versions -> endpoints: {}. Use -D{}}=true to ignore this, " +
+                    "-D{}=<ep1[,epN]> to skip specific endpoints, or -D{}=<ver1[,verN]> to skip specific schema versions",
+                    Schema.instance.getVersion(),
+                    migrationCoordinator.outstandingVersions(),
+                    CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK.getKey(),
+                    MigrationCoordinator.IGNORED_ENDPOINTS_PROP, MigrationCoordinator.IGNORED_VERSIONS_PROP);
+
+        if (requireSchemas)
+        {
+            logger.error("Didn't receive schemas for all known versions within the {}. Use -D{}=true to skip this check.",
+                         timeout, CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK.getKey());
+
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public void onRemove(InetAddressAndPort endpoint)
+    {
+        migrationCoordinator.removeAndIgnoreEndpoint(endpoint);
+    }
+
+    @Override
+    public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
+    {
+        if (state == ApplicationState.SCHEMA)
+        {
+            EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
+            if (epState != null && !Gossiper.instance.isDeadState(epState) && StorageService.instance.getTokenMetadata().isMember(endpoint))
+            {
+                migrationCoordinator.reportEndpointVersion(endpoint, UUID.fromString(value.value));
+            }
+        }
+    }
+
+    @Override
+    public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
+    {
+        // no-op
+    }
+
+    @Override
+    public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
+    {
+        // no-op
+    }
+
+    @Override
+    public void onAlive(InetAddressAndPort endpoint, EndpointState state)
+    {
+        // no-op
+    }
+
+    @Override
+    public void onDead(InetAddressAndPort endpoint, EndpointState state)
+    {
+        // no-op
+    }
+
+    @Override
+    public void onRestart(InetAddressAndPort endpoint, EndpointState state)
+    {
+        // no-op
+    }
+
+    private synchronized SchemaTransformationResult applyMutations(Collection<Mutation> schemaMutations)
+    {
+        // fetch the current state of schema for the affected keyspaces only
+        DistributedSchema before = schema;
+
+        // apply the schema mutations
+        SchemaKeyspace.applyChanges(schemaMutations);
+
+        // only compare the keyspaces affected by this set of schema mutations
+        Set<String> affectedKeyspaces = SchemaKeyspace.affectedKeyspaces(schemaMutations);
+
+        // apply the schema mutations and fetch the new versions of the altered keyspaces
+        Keyspaces updatedKeyspaces = SchemaKeyspace.fetchKeyspaces(affectedKeyspaces);
+        Set<String> removedKeyspaces = affectedKeyspaces.stream().filter(ks -> !updatedKeyspaces.containsKeyspace(ks)).collect(Collectors.toSet());
+        Keyspaces afterKeyspaces = before.getKeyspaces().withAddedOrReplaced(updatedKeyspaces).without(removedKeyspaces);
+
+        Keyspaces.KeyspacesDiff diff = Keyspaces.diff(before.getKeyspaces(), afterKeyspaces);
+        UUID version = SchemaKeyspace.calculateSchemaDigest();
+        DistributedSchema after = new DistributedSchema(afterKeyspaces, version);
+        SchemaTransformationResult update = new SchemaTransformationResult(before, after, diff);
+
+        updateSchema(update, false);
+        return update;
+    }
+
+    @Override
+    public synchronized SchemaTransformationResult apply(SchemaTransformation transformation, boolean local)
+    {
+        DistributedSchema before = schema;
+        Keyspaces afterKeyspaces = transformation.apply(before.getKeyspaces());
+        Keyspaces.KeyspacesDiff diff = Keyspaces.diff(before.getKeyspaces(), afterKeyspaces);
+
+        if (diff.isEmpty())
+            return new SchemaTransformationResult(before, before, diff);
+
+        Collection<Mutation> mutations = SchemaKeyspace.convertSchemaDiffToMutations(diff, transformation.fixedTimestampMicros().orElse(FBUtilities.timestampMicros()));
+        SchemaKeyspace.applyChanges(mutations);
+
+        DistributedSchema after = new DistributedSchema(afterKeyspaces, SchemaKeyspace.calculateSchemaDigest());
+        SchemaTransformationResult update = new SchemaTransformationResult(before, after, diff);
+
+        updateSchema(update, local);
+        if (!local)
+        {
+            migrationCoordinator.executor.submit(() -> {
+                Pair<Set<InetAddressAndPort>, Set<InetAddressAndPort>> endpoints = migrationCoordinator.pushSchemaMutations(mutations);
+                SchemaAnnouncementDiagnostics.schemaTransformationAnnounced(endpoints.left(), endpoints.right(), transformation);
+            });
+        }
+
+        return update;
+    }
+
+    private void updateSchema(SchemaTransformationResult update, boolean local)
+    {
+        this.schema = update.after;
+        logger.debug("Schema updated: {}", update);
+        updateCallback.accept(update, true);
+        if (!local)
+        {
+            migrationCoordinator.announce(update.after.getVersion());
+        }
+    }
+
+    private synchronized SchemaTransformationResult reload()
+    {
+        DistributedSchema before = this.schema;
+        DistributedSchema after = new DistributedSchema(SchemaKeyspace.fetchNonSystemKeyspaces(), SchemaKeyspace.calculateSchemaDigest());
+        Keyspaces.KeyspacesDiff diff = Keyspaces.diff(before.getKeyspaces(), after.getKeyspaces());
+        SchemaTransformationResult update = new SchemaTransformationResult(before, after, diff);
+
+        updateSchema(update, false);
+        return update;
+    }
+
+    @Override
+    public SchemaTransformationResult reset(boolean local)
+    {
+        return local
+               ? reload()
+               : migrationCoordinator.pullSchemaFromAnyNode()
+                                     .flatMap(mutations -> ImmediateFuture.success(applyMutations(mutations)))
+                                     .awaitThrowUncheckedOnInterrupt()
+                                     .getNow();
+    }
+
+    @Override
+    public synchronized void clear()
+    {
+        SchemaKeyspace.truncate();
+        this.schema = DistributedSchema.EMPTY;
+    }
+
+    private synchronized Collection<Mutation> getSchemaMutations()
+    {
+        return SchemaKeyspace.convertSchemaToMutations();
+    }
+
+    public Map<UUID, Set<InetAddressAndPort>> getOutstandingSchemaVersions()
+    {
+        return migrationCoordinator.outstandingVersions();
+    }
+}

src/java/org/apache/cassandra/schema/DefaultSchemaUpdateHandlerFactory.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.function.BiConsumer;
+
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
+
+public class DefaultSchemaUpdateHandlerFactory implements SchemaUpdateHandlerFactory
+{
+    public static final SchemaUpdateHandlerFactory instance = new DefaultSchemaUpdateHandlerFactory();
+
+    @Override
+    public SchemaUpdateHandler getSchemaUpdateHandler(boolean online, BiConsumer<SchemaTransformationResult, Boolean> updateSchemaCallback)
+    {
+        return online
+               ? new DefaultSchemaUpdateHandler(updateSchemaCallback)
+               : new OfflineSchemaUpdateHandler(updateSchemaCallback);
+    }
+}

src/java/org/apache/cassandra/schema/Diff.java

@@ -61,4 +61,15 @@ public class Diff<T extends Iterable, S>
             return String.format("%s -> %s (%s)", before, after, kind);
         }
     }
+
+    @Override
+    public String toString()
+    {
+        return "Diff{" +
+               "created=" + created +
+               ", dropped=" + dropped +
+               ", altered=" + altered +
+               '}';
+    }
+
 }

src/java/org/apache/cassandra/schema/DistributedSchema.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Objects;
+import java.util.UUID;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Immutable snapshot of the current schema along with its version.
+ */
+public class DistributedSchema
+{
+    public static final DistributedSchema EMPTY = new DistributedSchema(Keyspaces.none(), SchemaConstants.emptyVersion);
+
+    private final Keyspaces keyspaces;
+    private final UUID version;
+
+    public DistributedSchema(Keyspaces keyspaces, UUID version)
+    {
+        Objects.requireNonNull(keyspaces);
+        Objects.requireNonNull(version);
+        this.keyspaces = keyspaces;
+        this.version = version;
+        validate();
+    }
+
+    public Keyspaces getKeyspaces()
+    {
+        return keyspaces;
+    }
+
+    public boolean isEmpty()
+    {
+        return SchemaConstants.emptyVersion.equals(version);
+    }
+
+    public UUID getVersion()
+    {
+        return version;
+    }
+
+    /**
+     * Converts the given schema version to a string. Returns {@code unknown}, if {@code version} is {@code null}
+     * or {@code "(empty)"}, if {@code version} refers to an {@link SchemaConstants#emptyVersion empty) schema.
+     */
+    public static String schemaVersionToString(UUID version)
+    {
+        return version == null
+               ? "unknown"
+               : SchemaConstants.emptyVersion.equals(version)
+                 ? "(empty)"
+                 : version.toString();
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DistributedSchema schema = (DistributedSchema) o;
+        return keyspaces.equals(schema.keyspaces) && version.equals(schema.version);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(keyspaces, version);
+    }
+
+    private void validate()
+    {
+        keyspaces.forEach(ksm -> {
+            ksm.tables.forEach(tm -> Preconditions.checkArgument(tm.keyspace.equals(ksm.name), "Table %s metadata points to keyspace %s while defined in keyspace %s", tm.name, tm.keyspace, ksm.name));
+            ksm.views.forEach(vm -> Preconditions.checkArgument(vm.keyspace().equals(ksm.name), "View %s metadata points to keyspace %s while defined in keyspace %s", vm.name(), vm.keyspace(), ksm.name));
+            ksm.types.forEach(ut -> Preconditions.checkArgument(ut.keyspace.equals(ksm.name), "Type %s points to keyspace %s while defined in keyspace %s", ut.name, ut.keyspace, ksm.name));
+            ksm.functions.forEach(f -> Preconditions.checkArgument(f.name().keyspace.equals(ksm.name), "Function %s points to keyspace %s while defined in keyspace %s", f.name().name, f.name().keyspace, ksm.name));
+        });
+    }
+}

src/java/org/apache/cassandra/schema/KeyspaceMetadata.java

@@ -119,6 +119,11 @@ public final class KeyspaceMetadata implements SchemaElement
         return new KeyspaceMetadata(name, kind, params, tables, views, types, functions);
     }
 
+    public KeyspaceMetadata empty()
+    {
+        return new KeyspaceMetadata(this.name, this.kind, this.params, Tables.none(), Views.none(), Types.none(), Functions.none());
+    }
+
     public boolean isVirtual()
     {
         return kind == Kind.VIRTUAL;
@@ -391,5 +396,19 @@ public final class KeyspaceMetadata implements SchemaElement
 
             return Optional.of(new KeyspaceDiff(before, after, tables, views, types, udfs, udas));
         }
+
+        @Override
+        public String toString()
+        {
+            return "KeyspaceDiff{" +
+                   "before=" + before +
+                   ", after=" + after +
+                   ", tables=" + tables +
+                   ", views=" + views +
+                   ", types=" + types +
+                   ", udfs=" + udfs +
+                   ", udas=" + udas +
+                   '}';
+        }
     }
 }

src/java/org/apache/cassandra/schema/Keyspaces.java

@@ -17,9 +17,9 @@
  */
 package org.apache.cassandra.schema;
 
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
@@ -67,7 +67,7 @@ public final class Keyspaces implements Iterable<KeyspaceMetadata>
         return keyspaces.values().stream();
     }
 
-    public Set<String> names()
+    public ImmutableSet<String> names()
     {
         return keyspaces.keySet();
     }
@@ -124,6 +124,11 @@ public final class Keyspaces implements Iterable<KeyspaceMetadata>
         return filter(k -> k != keyspace);
     }
 
+    public Keyspaces without(Collection<String> names)
+    {
+        return filter(k -> !names.contains(k.name));
+    }
+
     public Keyspaces withAddedOrUpdated(KeyspaceMetadata keyspace)
     {
         return builder().add(Iterables.filter(this, k -> !k.name.equals(keyspace.name)))
@@ -131,6 +136,37 @@ public final class Keyspaces implements Iterable<KeyspaceMetadata>
                         .build();
     }
 
+    /**
+     * Returns a new {@link Keyspaces} equivalent to this one, but with the provided keyspace metadata either added (if
+     * this {@link Keyspaces} does not have that keyspace), or replaced by the provided definition.
+     *
+     * <p>Note that if this contains the provided keyspace, its pre-existing definition is discarded and completely
+     * replaced with the newly provided one. See {@link #withAddedOrUpdated(KeyspaceMetadata)} if you wish the provided
+     * definition to be "merged" with the existing one instead.
+     *
+     * @param keyspace the keyspace metadata to add, or replace the existing definition with.
+     * @return the newly created object.
+     */
+    public Keyspaces withAddedOrReplaced(KeyspaceMetadata keyspace)
+    {
+        return builder().add(Iterables.filter(this, k -> !k.name.equals(keyspace.name)))
+                        .add(keyspace)
+                        .build();
+    }
+
+    /**
+     * Calls {@link #withAddedOrReplaced(KeyspaceMetadata)} on all the keyspaces of the provided {@link Keyspaces}.
+     *
+     * @param keyspaces the keyspaces to add, or replace if existing.
+     * @return the newly created object.
+     */
+    public Keyspaces withAddedOrReplaced(Keyspaces keyspaces)
+    {
+        return builder().add(Iterables.filter(this, k -> !keyspaces.containsKeyspace(k.name)))
+                        .add(keyspaces)
+                        .build();
+    }
+
     public void validate()
     {
         keyspaces.values().forEach(KeyspaceMetadata::validate);
@@ -154,6 +190,11 @@ public final class Keyspaces implements Iterable<KeyspaceMetadata>
         return keyspaces.values().toString();
     }
 
+    public int size()
+    {
+        return keyspaces.size();
+    }
+
     public static final class Builder
     {
         private final ImmutableMap.Builder<String, KeyspaceMetadata> keyspaces = new ImmutableMap.Builder<>();
@@ -192,14 +233,14 @@ public final class Keyspaces implements Iterable<KeyspaceMetadata>
         }
     }
 
-    static KeyspacesDiff diff(Keyspaces before, Keyspaces after)
+    public static KeyspacesDiff diff(Keyspaces before, Keyspaces after)
     {
         return KeyspacesDiff.diff(before, after);
     }
 
     public static final class KeyspacesDiff
     {
-        static final KeyspacesDiff NONE = new KeyspacesDiff(Keyspaces.none(), Keyspaces.none(), ImmutableList.of());
+        public static final KeyspacesDiff NONE = new KeyspacesDiff(Keyspaces.none(), Keyspaces.none(), ImmutableList.of());
 
         public final Keyspaces created;
         public final Keyspaces dropped;
@@ -235,5 +276,15 @@ public final class Keyspaces implements Iterable<KeyspaceMetadata>
         {
             return created.isEmpty() && dropped.isEmpty() && altered.isEmpty();
         }
+
+        @Override
+        public String toString()
+        {
+            return "KeyspacesDiff{" +
+                   "created=" + created +
+                   ", dropped=" + dropped +
+                   ", altered=" + altered +
+                   '}';
+        }
     }
 }

src/java/org/apache/cassandra/schema/MigrationCoordinator.java

@@ -23,36 +23,38 @@ import java.net.UnknownHostException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.LongSupplier;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+import java.util.function.Supplier;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
-import org.apache.cassandra.concurrent.FutureTask;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.utils.Simulate;
-import org.apache.cassandra.utils.concurrent.Future;
-import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.FutureTask;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.EndpointState;
-import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
@@ -60,32 +62,36 @@ import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.NoPayload;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Simulate;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
 
+import static org.apache.cassandra.net.Verb.SCHEMA_PUSH_REQ;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.Simulate.With.MONITORS;
 import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue;
 
+/**
+ * Migration coordinator is responsible for tracking schema versions on various nodes and, if needed, synchronize the
+ * schema. It performs periodic checks and if there is a schema version mismatch between the current node and the other
+ * node, it pulls the schema and applies the changes locally through the callback.
+ *
+ * It works in close cooperation with {@link DefaultSchemaUpdateHandler} which is responsible for maintaining local
+ * schema metadata stored in {@link SchemaKeyspace}.
+ */
 @Simulate(with = MONITORS)
 public class MigrationCoordinator
 {
     private static final Logger logger = LoggerFactory.getLogger(MigrationCoordinator.class);
     private static final Future<Void> FINISHED_FUTURE = ImmediateFuture.success(null);
 
-    private static LongSupplier getUptimeFn = () -> ManagementFactory.getRuntimeMXBean().getUptime();
-
-    @VisibleForTesting
-    public static void setUptimeFn(LongSupplier supplier)
-    {
-        getUptimeFn = supplier;
-    }
-
-
     private static final int MIGRATION_DELAY_IN_MS = CassandraRelevantProperties.MIGRATION_DELAY.getInt();
-    private static final int MAX_OUTSTANDING_VERSION_REQUESTS = 3;
-
-    public static final MigrationCoordinator instance = new MigrationCoordinator();
+    public static final int MAX_OUTSTANDING_VERSION_REQUESTS = 3;
 
     public static final String IGNORED_VERSIONS_PROP = "cassandra.skip_schema_check_for_versions";
     public static final String IGNORED_ENDPOINTS_PROP = "cassandra.skip_schema_check_for_endpoints";
@@ -169,44 +175,68 @@ public class MigrationCoordinator
 
     private final Map<UUID, VersionInfo> versionInfo = new HashMap<>();
     private final Map<InetAddressAndPort, UUID> endpointVersions = new HashMap<>();
-    private final AtomicInteger inflightTasks = new AtomicInteger();
     private final Set<InetAddressAndPort> ignoredEndpoints = getIgnoredEndpoints();
+    private final ScheduledExecutorService periodicCheckExecutor;
+    private final MessagingService messagingService;
+    private final AtomicReference<ScheduledFuture<?>> periodicPullTask = new AtomicReference<>();
+    private final int maxOutstandingVersionRequests;
+    private final Gossiper gossiper;
+    private final Supplier<UUID> schemaVersion;
+    private final BiConsumer<InetAddressAndPort, Collection<Mutation>> schemaUpdateCallback;
 
-    public void start()
+    final ExecutorPlus executor;
+
+    /**
+     * Creates but does not start migration coordinator instance.
+     * @param messagingService      messaging service instance used to communicate with other nodes for pulling schema
+     *                              and pushing changes
+     * @param periodicCheckExecutor executor on which the periodic checks are scheduled
+     */
+    MigrationCoordinator(MessagingService messagingService,
+                         ExecutorPlus executor,
+                         ScheduledExecutorService periodicCheckExecutor,
+                         int maxOutstandingVersionRequests,
+                         Gossiper gossiper,
+                         Supplier<UUID> schemaVersionSupplier,
+                         BiConsumer<InetAddressAndPort, Collection<Mutation>> schemaUpdateCallback)
     {
-        ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(this::pullUnreceivedSchemaVersions, 1, 1, TimeUnit.MINUTES);
+        this.messagingService = messagingService;
+        this.executor = executor;
+        this.periodicCheckExecutor = periodicCheckExecutor;
+        this.maxOutstandingVersionRequests = maxOutstandingVersionRequests;
+        this.gossiper = gossiper;
+        this.schemaVersion = schemaVersionSupplier;
+        this.schemaUpdateCallback = schemaUpdateCallback;
     }
 
-    public synchronized void reset()
+    void start()
     {
-        versionInfo.clear();
+        announce(schemaVersion.get());
+        periodicPullTask.updateAndGet(curTask -> curTask == null
+                                                 ? periodicCheckExecutor.scheduleWithFixedDelay(this::pullUnreceivedSchemaVersions, 1, 1, TimeUnit.MINUTES)
+                                                 : curTask);
     }
 
-    synchronized List<Future<Void>> pullUnreceivedSchemaVersions()
+    private synchronized void pullUnreceivedSchemaVersions()
     {
-        List<Future<Void>> futures = new ArrayList<>();
         for (VersionInfo info : versionInfo.values())
         {
             if (info.wasReceived() || info.outstandingRequests.size() > 0)
                 continue;
 
-            Future<Void> future = maybePullSchema(info);
-            if (future != null && future != FINISHED_FUTURE)
-                futures.add(future);
+            maybePullSchema(info);
         }
-
-        return futures;
     }
 
-    synchronized Future<Void> maybePullSchema(VersionInfo info)
+    private synchronized Future<Void> maybePullSchema(VersionInfo info)
     {
         if (info.endpoints.isEmpty() || info.wasReceived() || !shouldPullSchema(info.version))
             return FINISHED_FUTURE;
 
-        if (info.outstandingRequests.size() >= getMaxOutstandingVersionRequests())
+        if (info.outstandingRequests.size() >= maxOutstandingVersionRequests)
             return FINISHED_FUTURE;
 
-        for (int i=0, isize=info.requestQueue.size(); i<isize; i++)
+        for (int i = 0, isize = info.requestQueue.size(); i < isize; i++)
         {
             InetAddressAndPort endpoint = info.requestQueue.remove();
             if (!info.endpoints.contains(endpoint))
@@ -224,10 +254,10 @@ public class MigrationCoordinator
         }
 
         // no suitable endpoints were found, check again in a minute, the periodic task will pick it up
-        return null;
+        return FINISHED_FUTURE;
     }
 
-    public synchronized Map<UUID, Set<InetAddressAndPort>> outstandingVersions()
+    synchronized Map<UUID, Set<InetAddressAndPort>> outstandingVersions()
     {
         HashMap<UUID, Set<InetAddressAndPort>> map = new HashMap<>();
         for (VersionInfo info : versionInfo.values())
@@ -237,58 +267,38 @@ public class MigrationCoordinator
     }
 
     @VisibleForTesting
-    protected VersionInfo getVersionInfoUnsafe(UUID version)
+    VersionInfo getVersionInfoUnsafe(UUID version)
     {
         return versionInfo.get(version);
     }
 
-    @VisibleForTesting
-    protected int getMaxOutstandingVersionRequests()
+    private boolean shouldPullSchema(UUID version)
     {
-        return MAX_OUTSTANDING_VERSION_REQUESTS;
-    }
-
-    @VisibleForTesting
-    protected boolean isAlive(InetAddressAndPort endpoint)
-    {
-        return FailureDetector.instance.isAlive(endpoint);
-    }
-
-    @VisibleForTesting
-    protected boolean shouldPullSchema(UUID version)
-    {
-        if (Schema.instance.getVersion() == null)
+        UUID localSchemaVersion = schemaVersion.get();
+        if (localSchemaVersion == null)
         {
             logger.debug("Not pulling schema for version {}, because local schama version is not known yet", version);
             return false;
         }
 
-        if (Schema.instance.isSameVersion(version))
+        if (localSchemaVersion.equals(version))
         {
             logger.debug("Not pulling schema for version {}, because schema versions match: " +
                          "local={}, remote={}",
                          version,
-                         Schema.schemaVersionToString(Schema.instance.getVersion()),
-                         Schema.schemaVersionToString(version));
+                         DistributedSchema.schemaVersionToString(localSchemaVersion),
+                         DistributedSchema.schemaVersionToString(version));
             return false;
         }
         return true;
     }
 
-    // Since 3.0.14 protocol contains only a CASSANDRA-13004 bugfix, it is safe to accept schema changes
-    // from both 3.0 and 3.0.14.
-    private static boolean is30Compatible(int version)
-    {
-        return version == MessagingService.current_version || version == MessagingService.VERSION_3014;
-    }
-
-    @VisibleForTesting
-    protected boolean shouldPullFromEndpoint(InetAddressAndPort endpoint)
+    private boolean shouldPullFromEndpoint(InetAddressAndPort endpoint)
     {
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return false;
 
-        EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
+        EndpointState state = gossiper.getEndpointStateForEndpoint(endpoint);
         if (state == null)
             return false;
 
@@ -302,19 +312,19 @@ public class MigrationCoordinator
             return false;
         }
 
-        if (!MessagingService.instance().versions.knows(endpoint))
+        if (!messagingService.versions.knows(endpoint))
         {
             logger.debug("Not pulling schema from {} because their messaging version is unknown", endpoint);
             return false;
         }
 
-        if (MessagingService.instance().versions.getRaw(endpoint) != MessagingService.current_version)
+        if (messagingService.versions.getRaw(endpoint) != MessagingService.current_version)
         {
             logger.debug("Not pulling schema from {} because their schema format is incompatible", endpoint);
             return false;
         }
 
-        if (Gossiper.instance.isGossipOnlyMember(endpoint))
+        if (gossiper.isGossipOnlyMember(endpoint))
         {
             logger.debug("Not pulling schema from {} because it's a gossip only member", endpoint);
             return false;
@@ -322,39 +332,33 @@ public class MigrationCoordinator
         return true;
     }
 
-    @VisibleForTesting
-    protected boolean shouldPullImmediately(InetAddressAndPort endpoint, UUID version)
+    private boolean shouldPullImmediately(InetAddressAndPort endpoint, UUID version)
     {
-        if (Schema.instance.isEmpty() || getUptimeFn.getAsLong() < MIGRATION_DELAY_IN_MS)
+        UUID localSchemaVersion = schemaVersion.get();
+        if (SchemaConstants.emptyVersion.equals(localSchemaVersion) || ManagementFactory.getRuntimeMXBean().getUptime() < MIGRATION_DELAY_IN_MS)
         {
             // If we think we may be bootstrapping or have recently started, submit MigrationTask immediately
             logger.debug("Immediately submitting migration task for {}, " +
                          "schema versions: local={}, remote={}",
                          endpoint,
-                         Schema.schemaVersionToString(Schema.instance.getVersion()),
-                         Schema.schemaVersionToString(version));
+                         DistributedSchema.schemaVersionToString(localSchemaVersion),
+                         DistributedSchema.schemaVersionToString(version));
             return true;
         }
         return false;
     }
 
-    @VisibleForTesting
-    protected boolean isLocalVersion(UUID version)
-    {
-        return Schema.instance.isSameVersion(version);
-    }
-
     /**
      * If a previous schema update brought our version the same as the incoming schema, don't apply it
      */
-    synchronized boolean shouldApplySchemaFor(VersionInfo info)
+    private synchronized boolean shouldApplySchemaFor(VersionInfo info)
     {
         if (info.wasReceived())
             return false;
-        return !isLocalVersion(info.version);
+        return !Objects.equals(schemaVersion.get(), info.version);
     }
 
-    public synchronized Future<Void> reportEndpointVersion(InetAddressAndPort endpoint, UUID version)
+    synchronized Future<Void> reportEndpointVersion(InetAddressAndPort endpoint, UUID version)
     {
         if (ignoredEndpoints.contains(endpoint) || IGNORED_VERSIONS.contains(version))
         {
@@ -368,7 +372,7 @@ public class MigrationCoordinator
             return FINISHED_FUTURE;
 
         VersionInfo info = versionInfo.computeIfAbsent(version, VersionInfo::new);
-        if (isLocalVersion(version))
+        if (Objects.equals(schemaVersion.get(), version))
             info.markReceived();
         info.endpoints.add(endpoint);
         info.requestQueue.addFirst(endpoint);
@@ -379,19 +383,6 @@ public class MigrationCoordinator
         return maybePullSchema(info);
     }
 
-    public Future<Void> reportEndpointVersion(InetAddressAndPort endpoint, EndpointState state)
-    {
-        if (state == null)
-            return FINISHED_FUTURE;
-
-        UUID version = state.getSchemaVersion();
-
-        if (version == null)
-            return FINISHED_FUTURE;
-
-        return reportEndpointVersion(endpoint, version);
-    }
-
     private synchronized void removeEndpointFromVersion(InetAddressAndPort endpoint, UUID version)
     {
         if (version == null)
@@ -410,7 +401,7 @@ public class MigrationCoordinator
         }
     }
 
-    public synchronized void removeAndIgnoreEndpoint(InetAddressAndPort endpoint)
+    synchronized void removeAndIgnoreEndpoint(InetAddressAndPort endpoint)
     {
         Preconditions.checkArgument(endpoint != null);
         ignoredEndpoints.add(endpoint);
@@ -421,38 +412,75 @@ public class MigrationCoordinator
         }
     }
 
-    Future<Void> scheduleSchemaPull(InetAddressAndPort endpoint, VersionInfo info)
+    private Future<Void> scheduleSchemaPull(InetAddressAndPort endpoint, VersionInfo info)
     {
-        FutureTask<Void> task = new FutureTask<>(() -> pullSchema(new Callback(endpoint, info)));
+        FutureTask<Void> task = new FutureTask<>(() -> pullSchema(endpoint, new Callback(endpoint, info)));
+
         if (shouldPullImmediately(endpoint, info.version))
-        {
             submitToMigrationIfNotShutdown(task);
-        }
         else
-        {
-            ScheduledExecutors.nonPeriodicTasks.schedule(()->submitToMigrationIfNotShutdown(task), MIGRATION_DELAY_IN_MS, TimeUnit.MILLISECONDS);
-        }
+            ScheduledExecutors.nonPeriodicTasks.schedule(() -> submitToMigrationIfNotShutdown(task), MIGRATION_DELAY_IN_MS, TimeUnit.MILLISECONDS);
+
         return task;
     }
 
-    private static Future<?> submitToMigrationIfNotShutdown(Runnable task)
+    private Future<Collection<Mutation>> pullSchemaFrom(InetAddressAndPort endpoint)
     {
-        if (Stage.MIGRATION.executor().isShutdown() || Stage.MIGRATION.executor().isTerminated())
+        AsyncPromise<Collection<Mutation>> result = new AsyncPromise<>();
+        return submitToMigrationIfNotShutdown(() -> pullSchema(endpoint, new RequestCallback<Collection<Mutation>>()
+        {
+            @Override
+            public void onResponse(Message<Collection<Mutation>> msg)
+            {
+                result.setSuccess(msg.payload);
+            }
+
+            @Override
+            public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+            {
+                result.setFailure(new RuntimeException("Failed to get schema from " + from + ". The failure reason was: " + failureReason));
+            }
+
+            @Override
+            public boolean invokeOnFailure()
+            {
+                return true;
+            }
+        })).flatMap(ignored -> result);
+    }
+
+    Future<Collection<Mutation>> pullSchemaFromAnyNode()
+    {
+        Optional<InetAddressAndPort> endpoint = gossiper.getLiveMembers()
+                                                        .stream()
+                                                        .filter(this::shouldPullFromEndpoint)
+                                                        .findFirst();
+
+        return endpoint.map(this::pullSchemaFrom).orElse(ImmediateFuture.success(Collections.emptyList()));
+    }
+
+
+    void announce(UUID schemaVersion)
+    {
+        if (gossiper.isEnabled())
+            gossiper.addLocalApplicationState(ApplicationState.SCHEMA, StorageService.instance.valueFactory.schema(schemaVersion));
+        SchemaDiagnostics.versionAnnounced(Schema.instance);
+    }
+
+    private Future<Void> submitToMigrationIfNotShutdown(Runnable task)
+    {
+        if (executor.isShutdown() || executor.isTerminated())
         {
             logger.info("Skipped scheduled pulling schema from other nodes: the MIGRATION executor service has been shutdown.");
-            return null;
+            return ImmediateFuture.success(null);
         }
         else
-            return Stage.MIGRATION.submit(task);
+        {
+            return executor.submit(task, null);
+        }
     }
 
-    @VisibleForTesting
-    protected void mergeSchemaFrom(InetAddressAndPort endpoint, Collection<Mutation> mutations)
-    {
-        Schema.instance.mergeAndAnnounceVersion(mutations);
-    }
-
-    class Callback implements RequestCallback<Collection<Mutation>>
+    private class Callback implements RequestCallback<Collection<Mutation>>
     {
         final InetAddressAndPort endpoint;
         final VersionInfo info;
@@ -486,7 +514,7 @@ public class MigrationCoordinator
                 {
                     try
                     {
-                        mergeSchemaFrom(endpoint, mutations);
+                        schemaUpdateCallback.accept(endpoint, mutations);
                     }
                     catch (Exception e)
                     {
@@ -504,40 +532,38 @@ public class MigrationCoordinator
         }
     }
 
-    private void pullSchema(Callback callback)
+    private void pullSchema(InetAddressAndPort endpoint, RequestCallback<Collection<Mutation>> callback)
     {
-        if (!isAlive(callback.endpoint))
+        if (!gossiper.isAlive(endpoint))
         {
-            logger.warn("Can't send schema pull request: node {} is down.", callback.endpoint);
-            callback.fail();
+            logger.warn("Can't send schema pull request: node {} is down.", endpoint);
+            callback.onFailure(endpoint, RequestFailureReason.UNKNOWN);
             return;
         }
 
         // There is a chance that quite some time could have passed between now and the MM#maybeScheduleSchemaPull(),
         // potentially enough for the endpoint node to restart - which is an issue if it does restart upgraded, with
         // a higher major.
-        if (!shouldPullFromEndpoint(callback.endpoint))
+        if (!shouldPullFromEndpoint(endpoint))
         {
-            logger.info("Skipped sending a migration request: node {} has a higher major version now.", callback.endpoint);
-            callback.fail();
+            logger.info("Skipped sending a migration request: node {} has a higher major version now.", endpoint);
+            callback.onFailure(endpoint, RequestFailureReason.UNKNOWN);
             return;
         }
 
-        logger.debug("Requesting schema from {}", callback.endpoint);
-        sendMigrationMessage(callback);
+        logger.debug("Requesting schema from {}", endpoint);
+        sendMigrationMessage(endpoint, callback);
     }
 
-    protected void sendMigrationMessage(Callback callback)
+    private void sendMigrationMessage(InetAddressAndPort endpoint, RequestCallback<Collection<Mutation>> callback)
     {
-        inflightTasks.getAndIncrement();
-        Message message = Message.out(Verb.SCHEMA_PULL_REQ, NoPayload.noPayload);
-        logger.info("Sending schema pull request to {}", callback.endpoint);
-        MessagingService.instance().sendWithCallback(message, callback.endpoint, callback);
+        Message<NoPayload> message = Message.out(Verb.SCHEMA_PULL_REQ, NoPayload.noPayload);
+        logger.info("Sending schema pull request to {}", endpoint);
+        messagingService.sendWithCallback(message, endpoint, callback);
     }
 
     private synchronized Future<Void> pullComplete(InetAddressAndPort endpoint, VersionInfo info, boolean wasSuccessful)
     {
-        inflightTasks.decrementAndGet();
         if (wasSuccessful)
             info.markReceived();
 
@@ -546,25 +572,18 @@ public class MigrationCoordinator
         return maybePullSchema(info);
     }
 
-    public int getInflightTasks()
-    {
-        return inflightTasks.get();
-    }
-
     /**
      * Wait until we've received schema responses for all versions we're aware of
      * @param waitMillis
      * @return true if response for all schemas were received, false if we timed out waiting
      */
-    public boolean awaitSchemaRequests(long waitMillis)
+    boolean awaitSchemaRequests(long waitMillis)
     {
         if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
             Gossiper.waitToSettle();
 
         if (versionInfo.isEmpty())
-        {
             logger.debug("Nothing in versionInfo - so no schemas to wait for");
-        }
 
         List<WaitQueue.Signal> signalList = null;
         try
@@ -593,4 +612,36 @@ public class MigrationCoordinator
                 signalList.forEach(WaitQueue.Signal::cancel);
         }
     }
+
+    Pair<Set<InetAddressAndPort>, Set<InetAddressAndPort>> pushSchemaMutations(Collection<Mutation> schemaMutations)
+    {
+        logger.debug("Pushing schema mutations: {}", schemaMutations);
+        Set<InetAddressAndPort> schemaDestinationEndpoints = new HashSet<>();
+        Set<InetAddressAndPort> schemaEndpointsIgnored = new HashSet<>();
+        Message<Collection<Mutation>> message = Message.out(SCHEMA_PUSH_REQ, schemaMutations);
+        for (InetAddressAndPort endpoint : gossiper.getLiveMembers())
+        {
+            if (shouldPushSchemaTo(endpoint))
+            {
+                logger.debug("Pushing schema mutations to {}: {}", endpoint, schemaMutations);
+                messagingService.send(message, endpoint);
+                schemaDestinationEndpoints.add(endpoint);
+            }
+            else
+            {
+                schemaEndpointsIgnored.add(endpoint);
+            }
+        }
+
+        return Pair.create(schemaDestinationEndpoints, schemaEndpointsIgnored);
+    }
+
+    private boolean shouldPushSchemaTo(InetAddressAndPort endpoint)
+    {
+        // only push schema to nodes with known and equal versions
+        return !endpoint.equals(FBUtilities.getBroadcastAddressAndPort())
+               && messagingService.versions.knows(endpoint)
+               && messagingService.versions.getRaw(endpoint) == MessagingService.current_version;
+    }
+
 }

src/java/org/apache/cassandra/schema/MigrationManager.java

@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.schema;
-
-import java.io.IOException;
-import java.util.*;
-import java.lang.management.ManagementFactory;
-import java.util.function.LongSupplier;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cassandra.utils.Simulate;
-import org.apache.cassandra.utils.concurrent.Future;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.exceptions.AlreadyExistsException;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.gms.*;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.concurrent.Stage.MIGRATION;
-import static org.apache.cassandra.net.Verb.SCHEMA_PUSH_REQ;
-import static org.apache.cassandra.utils.Simulate.With.GLOBAL_CLOCK;
-
-@Simulate(with = GLOBAL_CLOCK)
-public class MigrationManager
-{
-    private static final Logger logger = LoggerFactory.getLogger(MigrationManager.class);
-
-    public static final MigrationManager instance = new MigrationManager();
-
-    private static LongSupplier getUptimeFn = () -> ManagementFactory.getRuntimeMXBean().getUptime();
-
-    @VisibleForTesting
-    public static void setUptimeFn(LongSupplier supplier)
-    {
-        getUptimeFn = supplier;
-    }
-
-    private static final int MIGRATION_DELAY_IN_MS = 60000;
-
-    private static final int MIGRATION_TASK_WAIT_IN_SECONDS = Integer.parseInt(System.getProperty("cassandra.migration_task_wait_in_seconds", "1"));
-
-    private MigrationManager() {}
-
-    private static boolean shouldPushSchemaTo(InetAddressAndPort endpoint)
-    {
-        // only push schema to nodes with known and equal versions
-        return !endpoint.equals(FBUtilities.getBroadcastAddressAndPort())
-               && MessagingService.instance().versions.knows(endpoint)
-               && MessagingService.instance().versions.getRaw(endpoint) == MessagingService.current_version;
-    }
-
-    public static void announceNewKeyspace(KeyspaceMetadata ksm) throws ConfigurationException
-    {
-        announceNewKeyspace(ksm, false);
-    }
-
-    public static void announceNewKeyspace(KeyspaceMetadata ksm, boolean announceLocally) throws ConfigurationException
-    {
-        announceNewKeyspace(ksm, FBUtilities.timestampMicros(), announceLocally);
-    }
-
-    public static void announceNewKeyspace(KeyspaceMetadata ksm, long timestamp, boolean announceLocally) throws ConfigurationException
-    {
-        ksm.validate();
-
-        if (Schema.instance.getKeyspaceMetadata(ksm.name) != null)
-            throw new AlreadyExistsException(ksm.name);
-
-        logger.info("Create new Keyspace: {}", ksm);
-        announce(SchemaKeyspace.makeCreateKeyspaceMutation(ksm, timestamp), announceLocally);
-    }
-
-    public static void announceNewTable(TableMetadata cfm)
-    {
-        announceNewTable(cfm, true, FBUtilities.timestampMicros());
-    }
-
-    private static void announceNewTable(TableMetadata cfm, boolean throwOnDuplicate, long timestamp)
-    {
-        cfm.validate();
-
-        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(cfm.keyspace);
-        if (ksm == null)
-            throw new ConfigurationException(String.format("Cannot add table '%s' to non existing keyspace '%s'.", cfm.name, cfm.keyspace));
-        // If we have a table or a view which has the same name, we can't add a new one
-        else if (throwOnDuplicate && ksm.getTableOrViewNullable(cfm.name) != null)
-            throw new AlreadyExistsException(cfm.keyspace, cfm.name);
-
-        logger.info("Create new table: {}", cfm);
-        announce(SchemaKeyspace.makeCreateTableMutation(ksm, cfm, timestamp), false);
-    }
-
-    static void announceKeyspaceUpdate(KeyspaceMetadata ksm)
-    {
-        ksm.validate();
-
-        KeyspaceMetadata oldKsm = Schema.instance.getKeyspaceMetadata(ksm.name);
-        if (oldKsm == null)
-            throw new ConfigurationException(String.format("Cannot update non existing keyspace '%s'.", ksm.name));
-
-        logger.info("Update Keyspace '{}' From {} To {}", ksm.name, oldKsm, ksm);
-        announce(SchemaKeyspace.makeCreateKeyspaceMutation(ksm.name, ksm.params, FBUtilities.timestampMicros()), false);
-    }
-
-    public static void announceTableUpdate(TableMetadata tm)
-    {
-        announceTableUpdate(tm, false);
-    }
-
-    public static void announceTableUpdate(TableMetadata updated, boolean announceLocally)
-    {
-        updated.validate();
-
-        TableMetadata current = Schema.instance.getTableMetadata(updated.keyspace, updated.name);
-        if (current == null)
-            throw new ConfigurationException(String.format("Cannot update non existing table '%s' in keyspace '%s'.", updated.name, updated.keyspace));
-        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(current.keyspace);
-
-        updated.validateCompatibility(current);
-
-        long timestamp = FBUtilities.timestampMicros();
-
-        logger.info("Update table '{}/{}' From {} To {}", current.keyspace, current.name, current, updated);
-        Mutation.SimpleBuilder builder = SchemaKeyspace.makeUpdateTableMutation(ksm, current, updated, timestamp);
-
-        announce(builder, announceLocally);
-    }
-
-    static void announceKeyspaceDrop(String ksName)
-    {
-        KeyspaceMetadata oldKsm = Schema.instance.getKeyspaceMetadata(ksName);
-        if (oldKsm == null)
-            throw new ConfigurationException(String.format("Cannot drop non existing keyspace '%s'.", ksName));
-
-        logger.info("Drop Keyspace '{}'", oldKsm.name);
-        announce(SchemaKeyspace.makeDropKeyspaceMutation(oldKsm, FBUtilities.timestampMicros()), false);
-    }
-
-    public static void announceTableDrop(String ksName, String cfName, boolean announceLocally)
-    {
-        TableMetadata tm = Schema.instance.getTableMetadata(ksName, cfName);
-        if (tm == null)
-            throw new ConfigurationException(String.format("Cannot drop non existing table '%s' in keyspace '%s'.", cfName, ksName));
-        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(ksName);
-
-        logger.info("Drop table '{}/{}'", tm.keyspace, tm.name);
-        announce(SchemaKeyspace.makeDropTableMutation(ksm, tm, FBUtilities.timestampMicros()), announceLocally);
-    }
-
-    /**
-     * actively announce a new version to active hosts via rpc
-     * @param schema The schema mutation to be applied
-     */
-    private static void announce(Mutation.SimpleBuilder schema, boolean announceLocally)
-    {
-        List<Mutation> mutations = Collections.singletonList(schema.build());
-
-        if (announceLocally)
-            Schema.instance.merge(mutations);
-        else
-            announce(mutations);
-    }
-
-    public static void announce(Mutation change)
-    {
-        announce(Collections.singleton(change));
-    }
-
-    public static void announce(Collection<Mutation> schema)
-    {
-        Future<?> f = announceWithoutPush(schema);
-
-        Set<InetAddressAndPort> schemaDestinationEndpoints = new HashSet<>();
-        Set<InetAddressAndPort> schemaEndpointsIgnored = new HashSet<>();
-        Message<Collection<Mutation>> message = Message.out(SCHEMA_PUSH_REQ, schema);
-        for (InetAddressAndPort endpoint : Gossiper.instance.getLiveMembers())
-        {
-            if (shouldPushSchemaTo(endpoint))
-            {
-                MessagingService.instance().send(message, endpoint);
-                schemaDestinationEndpoints.add(endpoint);
-            }
-            else
-            {
-                schemaEndpointsIgnored.add(endpoint);
-            }
-        }
-
-        SchemaAnnouncementDiagnostics.schemaMutationsAnnounced(schemaDestinationEndpoints, schemaEndpointsIgnored);
-        FBUtilities.waitOnFuture(f);
-    }
-
-    public static Future<?> announceWithoutPush(Collection<Mutation> schema)
-    {
-        return MIGRATION.submit(() -> Schema.instance.mergeAndAnnounceVersion(schema));
-    }
-
-    public static KeyspacesDiff announce(SchemaTransformation transformation, boolean locally)
-    {
-        long now = FBUtilities.timestampMicros();
-
-        Future<Schema.TransformationResult> future =
-            MIGRATION.submit(() -> Schema.instance.transform(transformation, locally, now));
-
-        Schema.TransformationResult result = future.syncUninterruptibly().getNow();
-        if (!result.success)
-            throw result.exception;
-
-        if (locally || result.diff.isEmpty())
-            return result.diff;
-
-        Set<InetAddressAndPort> schemaDestinationEndpoints = new HashSet<>();
-        Set<InetAddressAndPort> schemaEndpointsIgnored = new HashSet<>();
-        Message<Collection<Mutation>> message = Message.out(SCHEMA_PUSH_REQ, result.mutations);
-        for (InetAddressAndPort endpoint : Gossiper.instance.getLiveMembers())
-        {
-            if (shouldPushSchemaTo(endpoint))
-            {
-                MessagingService.instance().send(message, endpoint);
-                schemaDestinationEndpoints.add(endpoint);
-            }
-            else
-            {
-                schemaEndpointsIgnored.add(endpoint);
-            }
-        }
-
-        SchemaAnnouncementDiagnostics.schemaTransformationAnnounced(schemaDestinationEndpoints, schemaEndpointsIgnored,
-                                                                    transformation);
-
-        return result.diff;
-    }
-
-    /**
-     * Clear all locally stored schema information and reset schema to initial state.
-     * Called by user (via JMX) who wants to get rid of schema disagreement.
-     */
-    public static void resetLocalSchema()
-    {
-        logger.info("Starting local schema reset...");
-
-        logger.debug("Truncating schema tables...");
-
-        SchemaMigrationDiagnostics.resetLocalSchema();
-
-        Schema.instance.truncateSchemaKeyspace();
-
-        logger.debug("Clearing local schema keyspace definitions...");
-
-        Schema.instance.clear();
-
-        Set<InetAddressAndPort> liveEndpoints = Gossiper.instance.getLiveMembers();
-        liveEndpoints.remove(FBUtilities.getBroadcastAddressAndPort());
-
-        // force migration if there are nodes around
-        for (InetAddressAndPort node : liveEndpoints)
-        {
-            EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(node);
-            Future<Void> pull = MigrationCoordinator.instance.reportEndpointVersion(node, state);
-            if (pull != null)
-                FBUtilities.waitOnFuture(pull);
-        }
-
-        logger.info("Local schema reset is complete.");
-    }
-
-    /**
-     * We have a set of non-local, distributed system keyspaces, e.g. system_traces, system_auth, etc.
-     * (see {@link SchemaConstants#REPLICATED_SYSTEM_KEYSPACE_NAMES}), that need to be created on cluster initialisation,
-     * and later evolved on major upgrades (sometimes minor too). This method compares the current known definitions
-     * of the tables (if the keyspace exists) to the expected, most modern ones expected by the running version of C*;
-     * if any changes have been detected, a schema Mutation will be created which, when applied, should make
-     * cluster's view of that keyspace aligned with the expected modern definition.
-     *
-     * @param keyspace   the expected modern definition of the keyspace
-     * @param generation timestamp to use for the table changes in the schema mutation
-     *
-     * @return empty Optional if the current definition is up to date, or an Optional with the Mutation that would
-     *         bring the schema in line with the expected definition.
-     */
-    public static Optional<Mutation> evolveSystemKeyspace(KeyspaceMetadata keyspace, long generation)
-    {
-        Mutation.SimpleBuilder builder = null;
-
-        KeyspaceMetadata definedKeyspace = Schema.instance.getKeyspaceMetadata(keyspace.name);
-        Tables definedTables = null == definedKeyspace ? Tables.none() : definedKeyspace.tables;
-
-        for (TableMetadata table : keyspace.tables)
-        {
-            if (table.equals(definedTables.getNullable(table.name)))
-                continue;
-
-            if (null == builder)
-            {
-                // for the keyspace definition itself (name, replication, durability) always use generation 0;
-                // this ensures that any changes made to replication by the user will never be overwritten.
-                builder = SchemaKeyspace.makeCreateKeyspaceMutation(keyspace.name, keyspace.params, 0);
-
-                // now set the timestamp to generation, so the tables have the expected timestamp
-                builder.timestamp(generation);
-            }
-
-            // for table definitions always use the provided generation; these tables, unlike their containing
-            // keyspaces, are *NOT* meant to be altered by the user; if their definitions need to change,
-            // the schema must be updated in code, and the appropriate generation must be bumped.
-            SchemaKeyspace.addTableToSchemaMutation(table, true, builder);
-        }
-
-        return builder == null ? Optional.empty() : Optional.of(builder.build());
-    }
-
-    public static class MigrationsSerializer implements IVersionedSerializer<Collection<Mutation>>
-    {
-        public static MigrationsSerializer instance = new MigrationsSerializer();
-
-        public void serialize(Collection<Mutation> schema, DataOutputPlus out, int version) throws IOException
-        {
-            out.writeInt(schema.size());
-            for (Mutation mutation : schema)
-                Mutation.serializer.serialize(mutation, out, version);
-        }
-
-        public Collection<Mutation> deserialize(DataInputPlus in, int version) throws IOException
-        {
-            int count = in.readInt();
-            Collection<Mutation> schema = new ArrayList<>(count);
-
-            for (int i = 0; i < count; i++)
-                schema.add(Mutation.serializer.deserialize(in, version));
-
-            return schema;
-        }
-
-        public long serializedSize(Collection<Mutation> schema, int version)
-        {
-            int size = TypeSizes.sizeof(schema.size());
-            for (Mutation mutation : schema)
-                size += mutation.serializedSize(version);
-            return size;
-        }
-    }
-}

src/java/org/apache/cassandra/schema/OfflineSchemaUpdateHandler.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.time.Duration;
+import java.util.UUID;
+import java.util.function.BiConsumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
+import org.apache.cassandra.utils.ByteArrayUtil;
+
+/**
+ * Update handler which works only in memory. It does not load or save the schema anywhere. It is used in client mode
+ * applications.
+ */
+public class OfflineSchemaUpdateHandler implements SchemaUpdateHandler
+{
+    private static final Logger logger = LoggerFactory.getLogger(OfflineSchemaUpdateHandler.class);
+
+    private final BiConsumer<SchemaTransformationResult, Boolean> updateCallback;
+
+    private volatile DistributedSchema schema = DistributedSchema.EMPTY;
+
+    public OfflineSchemaUpdateHandler(BiConsumer<SchemaTransformationResult, Boolean> updateCallback)
+    {
+        this.updateCallback = updateCallback;
+    }
+
+    @Override
+    public void start()
+    {
+        // no-op
+    }
+
+    @Override
+    public boolean waitUntilReady(Duration timeout)
+    {
+        return true;
+    }
+
+    @Override
+    public synchronized SchemaTransformationResult apply(SchemaTransformation transformation, boolean local)
+    {
+        DistributedSchema before = schema;
+        Keyspaces afterKeyspaces = transformation.apply(before.getKeyspaces());
+        Keyspaces.KeyspacesDiff diff = Keyspaces.diff(before.getKeyspaces(), afterKeyspaces);
+
+        if (diff.isEmpty())
+            return new SchemaTransformationResult(before, before, diff);
+
+        DistributedSchema after = new DistributedSchema(afterKeyspaces, UUID.nameUUIDFromBytes(ByteArrayUtil.bytes(afterKeyspaces.hashCode())));
+        SchemaTransformationResult update = new SchemaTransformationResult(before, after, diff);
+        this.schema = after;
+        logger.debug("Schema updated: {}", update);
+        updateCallback.accept(update, true);
+
+        return update;
+    }
+
+    @Override
+    public SchemaTransformationResult reset(boolean local)
+    {
+        if (!local)
+            throw new UnsupportedOperationException();
+
+        return apply(ignored -> SchemaKeyspace.fetchNonSystemKeyspaces(), local);
+    }
+
+    @Override
+    public synchronized void clear()
+    {
+        this.schema = DistributedSchema.EMPTY;
+    }
+}

src/java/org/apache/cassandra/schema/SchemaChangeListener.java

@@ -17,86 +17,94 @@
  */
 package org.apache.cassandra.schema;
 
-import java.util.List;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.db.marshal.UserType;
 
-import org.apache.cassandra.db.marshal.AbstractType;
-
-public abstract class SchemaChangeListener
+public interface SchemaChangeListener
 {
-    public void onCreateKeyspace(String keyspace)
+    default void onCreateKeyspace(KeyspaceMetadata keyspace)
     {
     }
 
-    public void onCreateTable(String keyspace, String table)
+    default void onCreateTable(TableMetadata table)
     {
     }
 
-    public void onCreateView(String keyspace, String view)
+    default void onCreateView(ViewMetadata view)
     {
-        onCreateTable(keyspace, view);
+        onCreateTable(view.metadata);
     }
 
-    public void onCreateType(String keyspace, String type)
+    default void onCreateType(UserType type)
     {
     }
 
-    public void onCreateFunction(String keyspace, String function, List<AbstractType<?>> argumentTypes)
+    default void onCreateFunction(UDFunction function)
     {
     }
 
-    public void onCreateAggregate(String keyspace, String aggregate, List<AbstractType<?>> argumentTypes)
+    default void onCreateAggregate(UDAggregate aggregate)
     {
     }
 
-    public void onAlterKeyspace(String keyspace)
+    default void onAlterKeyspace(KeyspaceMetadata before, KeyspaceMetadata after)
+    {
+    }
+
+    default void onPreAlterTable(TableMetadata before, TableMetadata after)
     {
     }
 
     // the boolean flag indicates whether the change that triggered this event may have a substantive
     // impact on statements using the column family.
-    public void onAlterTable(String keyspace, String table, boolean affectsStatements)
+    default void onAlterTable(TableMetadata before, TableMetadata after, boolean affectStatements)
     {
     }
 
-    public void onAlterView(String keyspace, String view, boolean affectsStataments)
-    {
-        onAlterTable(keyspace, view, affectsStataments);
-    }
-
-    public void onAlterType(String keyspace, String type)
+    default void onPreAlterView(ViewMetadata before, ViewMetadata after)
     {
     }
 
-    public void onAlterFunction(String keyspace, String function, List<AbstractType<?>> argumentTypes)
+    default void onAlterView(ViewMetadata before, ViewMetadata after, boolean affectStatements)
+    {
+        onAlterTable(before.metadata, after.metadata, affectStatements);
+    }
+
+    default void onAlterType(UserType before, UserType after)
     {
     }
 
-    public void onAlterAggregate(String keyspace, String aggregate, List<AbstractType<?>> argumentTypes)
+    default void onAlterFunction(UDFunction before, UDFunction after)
     {
     }
 
-    public void onDropKeyspace(String keyspace)
+    default void onAlterAggregate(UDAggregate before, UDAggregate after)
     {
     }
 
-    public void onDropTable(String keyspace, String table)
+    default void onDropKeyspace(KeyspaceMetadata keyspace)
     {
     }
 
-    public void onDropView(String keyspace, String view)
-    {
-        onDropTable(keyspace, view);
-    }
-
-    public void onDropType(String keyspace, String type)
+    default void onDropTable(TableMetadata table)
     {
     }
 
-    public void onDropFunction(String keyspace, String function, List<AbstractType<?>> argumentTypes)
+    default void onDropView(ViewMetadata view)
+    {
+        onDropTable(view.metadata);
+    }
+
+    default void onDropType(UserType type)
     {
     }
 
-    public void onDropAggregate(String keyspace, String aggregate, List<AbstractType<?>> argumentTypes)
+    default void onDropFunction(UDFunction function)
+    {
+    }
+
+    default void onDropAggregate(UDAggregate aggregate)
     {
     }
 }

src/java/org/apache/cassandra/schema/SchemaChangeNotifier.java

@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
+
+/**
+ * Registers schema change listeners and sends the notifications. The interface of this class just takes the high level
+ * keyspace metadata changes. It iterates over all keyspaces elements and distributes appropriate notifications about
+ * changes around those elements (tables, views, types, functions).
+ */
+public class SchemaChangeNotifier
+{
+    private final List<SchemaChangeListener> changeListeners = new CopyOnWriteArrayList<>();
+
+    public void registerListener(SchemaChangeListener listener)
+    {
+        changeListeners.add(listener);
+    }
+
+    @SuppressWarnings("unused")
+    public void unregisterListener(SchemaChangeListener listener)
+    {
+        changeListeners.remove(listener);
+    }
+
+    public void notifyKeyspaceCreated(KeyspaceMetadata keyspace)
+    {
+        notifyCreateKeyspace(keyspace);
+        keyspace.types.forEach(this::notifyCreateType);
+        keyspace.tables.forEach(this::notifyCreateTable);
+        keyspace.views.forEach(this::notifyCreateView);
+        keyspace.functions.udfs().forEach(this::notifyCreateFunction);
+        keyspace.functions.udas().forEach(this::notifyCreateAggregate);
+    }
+
+    public void notifyKeyspaceAltered(KeyspaceMetadata.KeyspaceDiff delta)
+    {
+        // notify on everything dropped
+        delta.udas.dropped.forEach(uda -> notifyDropAggregate((UDAggregate) uda));
+        delta.udfs.dropped.forEach(udf -> notifyDropFunction((UDFunction) udf));
+        delta.views.dropped.forEach(this::notifyDropView);
+        delta.tables.dropped.forEach(this::notifyDropTable);
+        delta.types.dropped.forEach(this::notifyDropType);
+
+        // notify on everything created
+        delta.types.created.forEach(this::notifyCreateType);
+        delta.tables.created.forEach(this::notifyCreateTable);
+        delta.views.created.forEach(this::notifyCreateView);
+        delta.udfs.created.forEach(udf -> notifyCreateFunction((UDFunction) udf));
+        delta.udas.created.forEach(uda -> notifyCreateAggregate((UDAggregate) uda));
+
+        // notify on everything altered
+        if (!delta.before.params.equals(delta.after.params))
+            notifyAlterKeyspace(delta.before, delta.after);
+        delta.types.altered.forEach(diff -> notifyAlterType(diff.before, diff.after));
+        delta.tables.altered.forEach(diff -> notifyAlterTable(diff.before, diff.after));
+        delta.views.altered.forEach(diff -> notifyAlterView(diff.before, diff.after));
+        delta.udfs.altered.forEach(diff -> notifyAlterFunction(diff.before, diff.after));
+        delta.udas.altered.forEach(diff -> notifyAlterAggregate(diff.before, diff.after));
+    }
+
+    public void notifyKeyspaceDropped(KeyspaceMetadata keyspace)
+    {
+        keyspace.functions.udas().forEach(this::notifyDropAggregate);
+        keyspace.functions.udfs().forEach(this::notifyDropFunction);
+        keyspace.views.forEach(this::notifyDropView);
+        keyspace.tables.forEach(this::notifyDropTable);
+        keyspace.types.forEach(this::notifyDropType);
+        notifyDropKeyspace(keyspace);
+    }
+
+    public void notifyPreChanges(SchemaTransformationResult transformationResult)
+    {
+        transformationResult.diff.altered.forEach(this::notifyPreAlterKeyspace);
+    }
+
+    private void notifyPreAlterKeyspace(KeyspaceMetadata.KeyspaceDiff keyspaceDiff)
+    {
+        keyspaceDiff.tables.altered.forEach(this::notifyPreAlterTable);
+        keyspaceDiff.views.altered.forEach(this::notifyPreAlterView);
+    }
+
+    private void notifyPreAlterTable(Diff.Altered<TableMetadata> altered)
+    {
+        changeListeners.forEach(l -> l.onPreAlterTable(altered.before, altered.after));
+    }
+
+    private void notifyPreAlterView(Diff.Altered<ViewMetadata> altered)
+    {
+        changeListeners.forEach(l -> l.onPreAlterView(altered.before, altered.after));
+    }
+
+    private void notifyCreateKeyspace(KeyspaceMetadata ksm)
+    {
+        changeListeners.forEach(l -> l.onCreateKeyspace(ksm));
+    }
+
+    private void notifyCreateTable(TableMetadata metadata)
+    {
+        changeListeners.forEach(l -> l.onCreateTable(metadata));
+    }
+
+    private void notifyCreateView(ViewMetadata view)
+    {
+        changeListeners.forEach(l -> l.onCreateView(view));
+    }
+
+    private void notifyCreateType(UserType ut)
+    {
+        changeListeners.forEach(l -> l.onCreateType(ut));
+    }
+
+    private void notifyCreateFunction(UDFunction udf)
+    {
+        changeListeners.forEach(l -> l.onCreateFunction(udf));
+    }
+
+    private void notifyCreateAggregate(UDAggregate udf)
+    {
+        changeListeners.forEach(l -> l.onCreateAggregate(udf));
+    }
+
+    private void notifyAlterKeyspace(KeyspaceMetadata before, KeyspaceMetadata after)
+    {
+        changeListeners.forEach(l -> l.onAlterKeyspace(before, after));
+    }
+
+    private void notifyAlterTable(TableMetadata before, TableMetadata after)
+    {
+        boolean changeAffectedPreparedStatements = before.changeAffectsPreparedStatements(after);
+        changeListeners.forEach(l -> l.onAlterTable(before, after, changeAffectedPreparedStatements));
+    }
+
+    private void notifyAlterView(ViewMetadata before, ViewMetadata after)
+    {
+        boolean changeAffectedPreparedStatements = before.metadata.changeAffectsPreparedStatements(after.metadata);
+        changeListeners.forEach(l -> l.onAlterView(before, after, changeAffectedPreparedStatements));
+    }
+
+    private void notifyAlterType(UserType before, UserType after)
+    {
+        changeListeners.forEach(l -> l.onAlterType(before, after));
+    }
+
+    private void notifyAlterFunction(UDFunction before, UDFunction after)
+    {
+        changeListeners.forEach(l -> l.onAlterFunction(before, after));
+    }
+
+    private void notifyAlterAggregate(UDAggregate before, UDAggregate after)
+    {
+        changeListeners.forEach(l -> l.onAlterAggregate(before, after));
+    }
+
+    private void notifyDropKeyspace(KeyspaceMetadata ksm)
+    {
+        changeListeners.forEach(l -> l.onDropKeyspace(ksm));
+    }
+
+    private void notifyDropTable(TableMetadata metadata)
+    {
+        changeListeners.forEach(l -> l.onDropTable(metadata));
+    }
+
+    private void notifyDropView(ViewMetadata view)
+    {
+        changeListeners.forEach(l -> l.onDropView(view));
+    }
+
+    private void notifyDropType(UserType ut)
+    {
+        changeListeners.forEach(l -> l.onDropType(ut));
+    }
+
+    private void notifyDropFunction(UDFunction udf)
+    {
+        changeListeners.forEach(l -> l.onDropFunction(udf));
+    }
+
+    private void notifyDropAggregate(UDAggregate udf)
+    {
+        changeListeners.forEach(l -> l.onDropAggregate(udf));
+    }
+}

src/java/org/apache/cassandra/schema/SchemaDiagnostics.java

@@ -86,28 +86,28 @@ final class SchemaDiagnostics
                                             delta.before, delta, null, null, null, null));
     }
 
-    static void keyspaceDroping(Schema schema, KeyspaceMetadata keyspace)
+    static void keyspaceDropping(Schema schema, KeyspaceMetadata keyspace)
     {
         if (isEnabled(SchemaEventType.KS_DROPPING))
             service.publish(new SchemaEvent(SchemaEventType.KS_DROPPING, schema, keyspace,
                                             null, null, null, null, null, null));
     }
 
-    static void keyspaceDroped(Schema schema, KeyspaceMetadata keyspace)
+    static void keyspaceDropped(Schema schema, KeyspaceMetadata keyspace)
     {
         if (isEnabled(SchemaEventType.KS_DROPPED))
             service.publish(new SchemaEvent(SchemaEventType.KS_DROPPED, schema, keyspace,
                                             null, null, null, null, null, null));
     }
 
-    static void schemataLoading(Schema schema)
+    static void schemaLoading(Schema schema)
     {
         if (isEnabled(SchemaEventType.SCHEMATA_LOADING))
             service.publish(new SchemaEvent(SchemaEventType.SCHEMATA_LOADING, schema, null,
                                             null, null, null, null, null, null));
     }
 
-    static void schemataLoaded(Schema schema)
+    static void schemaLoaded(Schema schema)
     {
         if (isEnabled(SchemaEventType.SCHEMATA_LOADED))
             service.publish(new SchemaEvent(SchemaEventType.SCHEMATA_LOADED, schema, null,
@@ -121,7 +121,7 @@ final class SchemaDiagnostics
                                             null, null, null, null, null, null));
     }
 
-    static void schemataCleared(Schema schema)
+    static void schemaCleared(Schema schema)
     {
         if (isEnabled(SchemaEventType.SCHEMATA_CLEARED))
             service.publish(new SchemaEvent(SchemaEventType.SCHEMATA_CLEARED, schema, null,

src/java/org/apache/cassandra/schema/SchemaEvent.java

@@ -21,29 +21,30 @@ package org.apache.cassandra.schema;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 import javax.annotation.Nullable;
 
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.MapDifference;
 
 import org.apache.cassandra.diag.DiagnosticEvent;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Collectors3;
 
 public final class SchemaEvent extends DiagnosticEvent
 {
     private final SchemaEventType type;
 
-    private final HashSet<String> keyspaces;
-    private final HashMap<String, String> indexTables;
-    private final HashMap<String, String> tables;
-    private final ArrayList<String> nonSystemKeyspaces;
-    private final ArrayList<String> userKeyspaces;
+    private final ImmutableCollection<String> keyspaces;
+    private final ImmutableMap<String, String> indexTables;
+    private final ImmutableCollection<String> tables;
+    private final ImmutableCollection<String> nonSystemKeyspaces;
+    private final ImmutableCollection<String> userKeyspaces;
     private final int numberOfTables;
     private final UUID version;
 
@@ -60,7 +61,7 @@ public final class SchemaEvent extends DiagnosticEvent
     @Nullable
     private final Views.ViewsDiff viewsDiff;
     @Nullable
-    private final MapDifference<String,TableMetadata> indexesDiff;
+    private final MapDifference<String, TableMetadata> indexesDiff;
 
     public enum SchemaEventType
     {
@@ -89,7 +90,7 @@ public final class SchemaEvent extends DiagnosticEvent
     SchemaEvent(SchemaEventType type, Schema schema, @Nullable KeyspaceMetadata ksUpdate,
                 @Nullable KeyspaceMetadata previous, @Nullable KeyspaceMetadata.KeyspaceDiff ksDiff,
                 @Nullable TableMetadata tableUpdate, @Nullable Tables.TablesDiff tablesDiff,
-                @Nullable Views.ViewsDiff viewsDiff, @Nullable MapDifference<String,TableMetadata> indexesDiff)
+                @Nullable Views.ViewsDiff viewsDiff, @Nullable MapDifference<String, TableMetadata> indexesDiff)
     {
         this.type = type;
         this.ksUpdate = ksUpdate;
@@ -100,27 +101,21 @@ public final class SchemaEvent extends DiagnosticEvent
         this.viewsDiff = viewsDiff;
         this.indexesDiff = indexesDiff;
 
-        this.keyspaces = new HashSet<>(schema.getKeyspaces());
-        this.nonSystemKeyspaces = new ArrayList<>(schema.getNonSystemKeyspaces());
-        this.userKeyspaces = new ArrayList<>(schema.getUserKeyspaces());
+        this.keyspaces = schema.distributedAndLocalKeyspaces().names();
+        this.nonSystemKeyspaces = schema.distributedKeyspaces().names();
+        this.userKeyspaces = schema.getUserKeyspaces().names();
         this.numberOfTables = schema.getNumberOfTables();
         this.version = schema.getVersion();
 
-        Map<Pair<String, String>, TableMetadataRef> indexTableMetadataRefs = schema.getIndexTableMetadataRefs();
-        Map<String, String> indexTables = indexTableMetadataRefs.entrySet().stream()
-                                                                .collect(Collectors.toMap(e -> e.getKey().left + ',' +
-                                                                                               e.getKey().right,
-                                                                                          e -> e.getValue().id.toHexString() + ',' +
-                                                                                               e.getValue().keyspace + ',' +
-                                                                                               e.getValue().name));
-        this.indexTables = new HashMap<>(indexTables);
-        Map<TableId, TableMetadataRef> tableMetadataRefs = schema.getTableMetadataRefs();
-        Map<String, String> tables = tableMetadataRefs.entrySet().stream()
-                                                      .collect(Collectors.toMap(e -> e.getKey().toHexString(),
-                                                                                e -> e.getValue().id.toHexString() + ',' +
-                                                                                     e.getValue().keyspace + ',' +
-                                                                                     e.getValue().name));
-        this.tables = new HashMap<>(tables);
+        this.indexTables = schema.distributedKeyspaces().stream()
+                                 .flatMap(ks -> ks.tables.indexTables().entrySet().stream())
+                                 .collect(Collectors3.toImmutableMap(e -> String.format("%s,%s", e.getValue().keyspace, e.getKey()),
+                                                                     e -> String.format("%s,%s,%s", e.getValue().id.toHexString(), e.getValue().keyspace, e.getValue().name)));
+
+        this.tables = schema.distributedKeyspaces().stream()
+                            .flatMap(ks -> StreamSupport.stream(ks.tablesAndViews().spliterator(), false))
+                            .map(e -> String.format("%s,%s,%s", e.id.toHexString(), e.keyspace, e.name))
+                            .collect(Collectors3.toImmutableList());
     }
 
     public SchemaEventType getType()

src/java/org/apache/cassandra/schema/SchemaKeyspace.java

@@ -64,7 +64,8 @@ import static org.apache.cassandra.utils.Simulate.With.GLOBAL_CLOCK;
 /**
  * system_schema.* tables and methods for manipulating them.
  * 
- * Please notice this class is _not_ thread safe. It should be accessed through {@link org.apache.cassandra.schema.Schema}. See CASSANDRA-16856/16996
+ * Please notice this class is _not_ thread safe and all methods which reads or updates the data in schema keyspace
+ * should be accessed only from the implementation of {@link SchemaUpdateHandler} in synchronized blocks.
  */
 @NotThreadSafe
 public final class SchemaKeyspace
@@ -250,7 +251,7 @@ public final class SchemaKeyspace
                                    .build();
     }
 
-    static KeyspaceMetadata metadata()
+    public static KeyspaceMetadata metadata()
     {
         return KeyspaceMetadata.create(SchemaConstants.SCHEMA_KEYSPACE_NAME, KeyspaceParams.local(), org.apache.cassandra.schema.Tables.of(ALL_TABLE_METADATA));
     }
@@ -259,32 +260,32 @@ public final class SchemaKeyspace
     {
         Map<String, Mutation> mutations = new HashMap<>();
 
-        diff.created.forEach(k -> mutations.put(k.name, makeCreateKeyspaceMutation(k, timestamp).build()));
         diff.dropped.forEach(k -> mutations.put(k.name, makeDropKeyspaceMutation(k, timestamp).build()));
+        diff.created.forEach(k -> mutations.put(k.name, makeCreateKeyspaceMutation(k, timestamp).build()));
         diff.altered.forEach(kd ->
         {
             KeyspaceMetadata ks = kd.after;
 
             Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(ks.name, ks.params, timestamp);
 
-            kd.types.created.forEach(t -> addTypeToSchemaMutation(t, builder));
             kd.types.dropped.forEach(t -> addDropTypeToSchemaMutation(t, builder));
+            kd.types.created.forEach(t -> addTypeToSchemaMutation(t, builder));
             kd.types.altered(Difference.SHALLOW).forEach(td -> addTypeToSchemaMutation(td.after, builder));
 
-            kd.tables.created.forEach(t -> addTableToSchemaMutation(t, true, builder));
             kd.tables.dropped.forEach(t -> addDropTableToSchemaMutation(t, builder));
+            kd.tables.created.forEach(t -> addTableToSchemaMutation(t, true, builder));
             kd.tables.altered(Difference.SHALLOW).forEach(td -> addAlterTableToSchemaMutation(td.before, td.after, builder));
 
-            kd.views.created.forEach(v -> addViewToSchemaMutation(v, true, builder));
             kd.views.dropped.forEach(v -> addDropViewToSchemaMutation(v, builder));
+            kd.views.created.forEach(v -> addViewToSchemaMutation(v, true, builder));
             kd.views.altered(Difference.SHALLOW).forEach(vd -> addAlterViewToSchemaMutation(vd.before, vd.after, builder));
 
-            kd.udfs.created.forEach(f -> addFunctionToSchemaMutation((UDFunction) f, builder));
             kd.udfs.dropped.forEach(f -> addDropFunctionToSchemaMutation((UDFunction) f, builder));
+            kd.udfs.created.forEach(f -> addFunctionToSchemaMutation((UDFunction) f, builder));
             kd.udfs.altered(Difference.SHALLOW).forEach(fd -> addFunctionToSchemaMutation(fd.after, builder));
 
-            kd.udas.created.forEach(a -> addAggregateToSchemaMutation((UDAggregate) a, builder));
             kd.udas.dropped.forEach(a -> addDropAggregateToSchemaMutation((UDAggregate) a, builder));
+            kd.udas.created.forEach(a -> addAggregateToSchemaMutation((UDAggregate) a, builder));
             kd.udas.altered(Difference.SHALLOW).forEach(ad -> addAggregateToSchemaMutation(ad.after, builder));
 
             mutations.put(ks.name, builder.build());
@@ -319,6 +320,7 @@ public final class SchemaKeyspace
 
     static void truncate()
     {
+        logger.debug("Truncating schema tables...");
         ALL.reverse().forEach(table -> getSchemaCFS(table).truncateBlocking());
     }
 
@@ -332,7 +334,7 @@ public final class SchemaKeyspace
      * Read schema from system keyspace and calculate MD5 digest of every row, resulting digest
      * will be converted into UUID which would act as content-based version of the schema.
      */
-    static UUID calculateSchemaDigest()
+    public static UUID calculateSchemaDigest()
     {
         Digest digest = Digest.forSchema();
         for (String table : ALL)
@@ -444,7 +446,7 @@ public final class SchemaKeyspace
         return metadata.partitioner.decorateKey(metadata.partitionKeyType.decomposeUntyped(value));
     }
 
-    static Mutation.SimpleBuilder makeCreateKeyspaceMutation(String name, KeyspaceParams params, long timestamp)
+    private static Mutation.SimpleBuilder makeCreateKeyspaceMutation(String name, KeyspaceParams params, long timestamp)
     {
         Mutation.SimpleBuilder builder = Mutation.simpleBuilder(Keyspaces.keyspace, decorate(Keyspaces, name))
                                                  .timestamp(timestamp);
@@ -457,6 +459,7 @@ public final class SchemaKeyspace
         return builder;
     }
 
+    @VisibleForTesting
     static Mutation.SimpleBuilder makeCreateKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
     {
         Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
@@ -470,7 +473,7 @@ public final class SchemaKeyspace
         return builder;
     }
 
-    static Mutation.SimpleBuilder makeDropKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
+    private static Mutation.SimpleBuilder makeDropKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp)
     {
         Mutation.SimpleBuilder builder = Mutation.simpleBuilder(SchemaConstants.SCHEMA_KEYSPACE_NAME, decorate(Keyspaces, keyspace.name))
                                                  .timestamp(timestamp);
@@ -494,6 +497,7 @@ public final class SchemaKeyspace
         builder.update(Types).row(type.name).delete();
     }
 
+    @VisibleForTesting
     static Mutation.SimpleBuilder makeCreateTableMutation(KeyspaceMetadata keyspace, TableMetadata table, long timestamp)
     {
         // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
@@ -502,7 +506,7 @@ public final class SchemaKeyspace
         return builder;
     }
 
-    static void addTableToSchemaMutation(TableMetadata table, boolean withColumnsAndTriggers, Mutation.SimpleBuilder builder)
+    private static void addTableToSchemaMutation(TableMetadata table, boolean withColumnsAndTriggers, Mutation.SimpleBuilder builder)
     {
         Row.SimpleBuilder rowBuilder = builder.update(Tables)
                                               .row(table.name)
@@ -608,6 +612,7 @@ public final class SchemaKeyspace
             addUpdatedIndexToSchemaMutation(newTable, diff.rightValue(), builder);
     }
 
+    @VisibleForTesting
     static Mutation.SimpleBuilder makeUpdateTableMutation(KeyspaceMetadata keyspace,
                                                           TableMetadata oldTable,
                                                           TableMetadata newTable,
@@ -640,14 +645,6 @@ public final class SchemaKeyspace
         return Maps.difference(beforeMap, afterMap);
     }
 
-    static Mutation.SimpleBuilder makeDropTableMutation(KeyspaceMetadata keyspace, TableMetadata table, long timestamp)
-    {
-        // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
-        Mutation.SimpleBuilder builder = makeCreateKeyspaceMutation(keyspace.name, keyspace.params, timestamp);
-        addDropTableToSchemaMutation(table, builder);
-        return builder;
-    }
-
     private static void addDropTableToSchemaMutation(TableMetadata table, Mutation.SimpleBuilder builder)
     {
         builder.update(Tables).row(table.name).delete();
@@ -946,6 +943,7 @@ public final class SchemaKeyspace
                             .build();
     }
 
+    @VisibleForTesting
     static TableParams createTableParamsFromRow(UntypedResultSet.Row row)
     {
         return TableParams.builder()
@@ -986,6 +984,7 @@ public final class SchemaKeyspace
         return columns;
     }
 
+    @VisibleForTesting
     static ColumnMetadata createColumnFromRow(UntypedResultSet.Row row, Types types)
     {
         String keyspace = row.getString("keyspace_name");
@@ -1032,7 +1031,7 @@ public final class SchemaKeyspace
                                  ? ColumnMetadata.Kind.valueOf(row.getString("kind").toUpperCase())
                                  : ColumnMetadata.Kind.REGULAR;
         assert kind == ColumnMetadata.Kind.REGULAR || kind == ColumnMetadata.Kind.STATIC
-            : "Unexpected dropped column kind: " + kind.toString();
+            : "Unexpected dropped column kind: " + kind;
 
         ColumnMetadata column = new ColumnMetadata(keyspace, table, ColumnIdentifier.getInterned(name, true), type, ColumnMetadata.NO_POSITION, kind);
         long droppedTime = TimeUnit.MILLISECONDS.toMicros(row.getLong("dropped_time"));

src/java/org/apache/cassandra/schema/SchemaMigrationDiagnostics.java

@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.schema;
-
-import java.util.UUID;
-
-import org.apache.cassandra.diag.DiagnosticEventService;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.schema.SchemaMigrationEvent.MigrationManagerEventType;
-
-final class SchemaMigrationDiagnostics
-{
-    private static final DiagnosticEventService service = DiagnosticEventService.instance();
-
-    private SchemaMigrationDiagnostics()
-    {
-    }
-
-    static void unknownLocalSchemaVersion(InetAddressAndPort endpoint, UUID theirVersion)
-    {
-        if (isEnabled(MigrationManagerEventType.UNKNOWN_LOCAL_SCHEMA_VERSION))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.UNKNOWN_LOCAL_SCHEMA_VERSION, endpoint,
-                                                     theirVersion));
-    }
-
-    static void versionMatch(InetAddressAndPort endpoint, UUID theirVersion)
-    {
-        if (isEnabled(MigrationManagerEventType.VERSION_MATCH))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.VERSION_MATCH, endpoint, theirVersion));
-    }
-
-    static void skipPull(InetAddressAndPort endpoint, UUID theirVersion)
-    {
-        if (isEnabled(MigrationManagerEventType.SKIP_PULL))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.SKIP_PULL, endpoint, theirVersion));
-    }
-
-    static void resetLocalSchema()
-    {
-        if (isEnabled(MigrationManagerEventType.RESET_LOCAL_SCHEMA))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.RESET_LOCAL_SCHEMA, null, null));
-    }
-
-    static void taskCreated(InetAddressAndPort endpoint)
-    {
-        if (isEnabled(MigrationManagerEventType.TASK_CREATED))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.TASK_CREATED, endpoint, null));
-    }
-
-    static void taskSendAborted(InetAddressAndPort endpoint)
-    {
-        if (isEnabled(MigrationManagerEventType.TASK_SEND_ABORTED))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.TASK_SEND_ABORTED, endpoint, null));
-    }
-
-    static void taskRequestSend(InetAddressAndPort endpoint)
-    {
-        if (isEnabled(MigrationManagerEventType.TASK_REQUEST_SEND))
-            service.publish(new SchemaMigrationEvent(MigrationManagerEventType.TASK_REQUEST_SEND,
-                                                     endpoint, null));
-    }
-
-    private static boolean isEnabled(MigrationManagerEventType type)
-    {
-        return service.isEnabled(SchemaMigrationEvent.class, type);
-    }
-}

src/java/org/apache/cassandra/schema/SchemaMigrationEvent.java

@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.schema;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.annotation.Nullable;
-
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.diag.DiagnosticEvent;
-import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.MessagingService;
-
-/**
- * Internal events emitted by {@link MigrationManager}.
- */
-final class SchemaMigrationEvent extends DiagnosticEvent
-{
-    private final MigrationManagerEventType type;
-    @Nullable
-    private final InetAddressAndPort endpoint;
-    @Nullable
-    private final UUID endpointSchemaVersion;
-    private final UUID localSchemaVersion;
-    private final Integer localMessagingVersion;
-    private final SystemKeyspace.BootstrapState bootstrapState;
-    private final Integer inflightTaskCount;
-    @Nullable
-    private Integer endpointMessagingVersion;
-    @Nullable
-    private Boolean endpointGossipOnlyMember;
-    @Nullable
-    private Boolean isAlive;
-
-    enum MigrationManagerEventType
-    {
-        UNKNOWN_LOCAL_SCHEMA_VERSION,
-        VERSION_MATCH,
-        SKIP_PULL,
-        RESET_LOCAL_SCHEMA,
-        TASK_CREATED,
-        TASK_SEND_ABORTED,
-        TASK_REQUEST_SEND
-    }
-
-    SchemaMigrationEvent(MigrationManagerEventType type,
-                         @Nullable InetAddressAndPort endpoint, @Nullable UUID endpointSchemaVersion)
-    {
-        this.type = type;
-        this.endpoint = endpoint;
-        this.endpointSchemaVersion = endpointSchemaVersion;
-
-        localSchemaVersion = Schema.instance.getVersion();
-        localMessagingVersion = MessagingService.current_version;
-
-        inflightTaskCount = MigrationCoordinator.instance.getInflightTasks();
-
-        this.bootstrapState = SystemKeyspace.getBootstrapState();
-
-        if (endpoint == null) return;
-
-        if (MessagingService.instance().versions.knows(endpoint))
-            endpointMessagingVersion = MessagingService.instance().versions.getRaw(endpoint);
-
-        endpointGossipOnlyMember = Gossiper.instance.isGossipOnlyMember(endpoint);
-        this.isAlive = FailureDetector.instance.isAlive(endpoint);
-    }
-
-    public Enum<?> getType()
-    {
-        return type;
-    }
-
-    public Map<String, Serializable> toMap()
-    {
-        HashMap<String, Serializable> ret = new HashMap<>();
-        if (endpoint != null) ret.put("endpoint", endpoint.getHostAddressAndPort());
-        ret.put("endpointSchemaVersion", Schema.schemaVersionToString(endpointSchemaVersion));
-        ret.put("localSchemaVersion", Schema.schemaVersionToString(localSchemaVersion));
-        if (endpointMessagingVersion != null) ret.put("endpointMessagingVersion", endpointMessagingVersion);
-        if (localMessagingVersion != null) ret.put("localMessagingVersion", localMessagingVersion);
-        if (endpointGossipOnlyMember != null) ret.put("endpointGossipOnlyMember", endpointGossipOnlyMember);
-        if (isAlive != null) ret.put("endpointIsAlive", isAlive);
-        if (bootstrapState != null) ret.put("bootstrapState", bootstrapState.name());
-        if (inflightTaskCount != null) ret.put("inflightTaskCount", inflightTaskCount);
-        return ret;
-    }
-}

src/java/org/apache/cassandra/schema/SchemaMutationsSerializer.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class SchemaMutationsSerializer implements IVersionedSerializer<Collection<Mutation>>
+{
+    public static final SchemaMutationsSerializer instance = new SchemaMutationsSerializer();
+
+    public void serialize(Collection<Mutation> schema, DataOutputPlus out, int version) throws IOException
+    {
+        out.writeInt(schema.size());
+        for (Mutation mutation : schema)
+            Mutation.serializer.serialize(mutation, out, version);
+    }
+
+    public Collection<Mutation> deserialize(DataInputPlus in, int version) throws IOException
+    {
+        int count = in.readInt();
+        Collection<Mutation> schema = new ArrayList<>(count);
+
+        for (int i = 0; i < count; i++)
+            schema.add(Mutation.serializer.deserialize(in, version));
+
+        return schema;
+    }
+
+    public long serializedSize(Collection<Mutation> schema, int version)
+    {
+        int size = TypeSizes.sizeof(schema.size());
+        for (Mutation mutation : schema)
+            size += mutation.serializedSize(version);
+        return size;
+    }
+}

src/java/org/apache/cassandra/schema/SchemaPullVerbHandler.java

@@ -18,11 +18,16 @@
 package org.apache.cassandra.schema;
 
 import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Consumer;
+import java.util.function.Function;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
@@ -38,10 +43,20 @@ public final class SchemaPullVerbHandler implements IVerbHandler<NoPayload>
 
     private static final Logger logger = LoggerFactory.getLogger(SchemaPullVerbHandler.class);
 
+    private final List<Consumer<Message<NoPayload>>> handlers = new CopyOnWriteArrayList<>();
+
+    public void register(Consumer<Message<NoPayload>> handler)
+    {
+        handlers.add(handler);
+    }
+
     public void doVerb(Message<NoPayload> message)
     {
         logger.trace("Received schema pull request from {}", message.from());
-        Message<Collection<Mutation>> response = message.responseWith(Schema.instance.schemaKeyspaceAsMutations());
-        MessagingService.instance().send(response, message.from());
+        List<Consumer<Message<NoPayload>>> handlers = this.handlers;
+        if (handlers.isEmpty())
+            throw new UnsupportedOperationException("There is no handler registered for schema pull verb");
+
+        handlers.forEach(h -> h.accept(message));
     }
 }

src/java/org/apache/cassandra/schema/SchemaPushVerbHandler.java

@@ -18,6 +18,9 @@
 package org.apache.cassandra.schema;
 
 import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Consumer;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,6 +29,7 @@ import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.NoPayload;
 
 /**
  * Called when node receives updated schema state from the schema migration coordinator node.
@@ -39,11 +43,22 @@ public final class SchemaPushVerbHandler implements IVerbHandler<Collection<Muta
 
     private static final Logger logger = LoggerFactory.getLogger(SchemaPushVerbHandler.class);
 
+    private final List<Consumer<Message<Collection<Mutation>>>> handlers = new CopyOnWriteArrayList<>();
+
+    public void register(Consumer<Message<Collection<Mutation>>> handler)
+    {
+        handlers.add(handler);
+    }
+
     public void doVerb(final Message<Collection<Mutation>> message)
     {
         logger.trace("Received schema push request from {}", message.from());
-
         SchemaAnnouncementDiagnostics.schemataMutationsReceived(message.from());
-        Stage.MIGRATION.submit(() -> Schema.instance.mergeAndAnnounceVersion(message.payload));
+
+        List<Consumer<Message<Collection<Mutation>>>> handlers = this.handlers;
+        if (handlers.isEmpty())
+            throw new UnsupportedOperationException("There is no handler registered for schema push verb");
+
+        handlers.forEach(h -> h.accept(message));
     }
 }

src/java/org/apache/cassandra/schema/SchemaTransformation.java

@@ -17,17 +17,51 @@
  */
 package org.apache.cassandra.schema;
 
-import java.net.UnknownHostException;
+import java.util.Optional;
 
 public interface SchemaTransformation
 {
     /**
      * Apply a statement transformation to a schema snapshot.
-     *
-     * Implementing methods should be side-effect free.
+     * <p>
+     * Implementing methods should be side-effect free (outside of throwing exceptions if the transformation cannot
+     * be successfully applied to the provided schema).
      *
      * @param schema Keyspaces to base the transformation on
      * @return Keyspaces transformed by the statement
      */
-    Keyspaces apply(Keyspaces schema) throws UnknownHostException;
+    Keyspaces apply(Keyspaces schema);
+
+    /**
+     * If the transformation should be applied with a certain timestamp, this method should be overriden. This is used
+     * by {@link SchemaTransformations#updateSystemKeyspace(KeyspaceMetadata, long)} when we need to set the fixed
+     * timestamp in order to preserve user settings.
+     */
+    default Optional<Long> fixedTimestampMicros()
+    {
+        return Optional.empty();
+    }
+
+    /**
+     * The result of applying (on this node) a given schema transformation.
+     */
+    class SchemaTransformationResult
+    {
+        public final DistributedSchema before;
+        public final DistributedSchema after;
+        public final Keyspaces.KeyspacesDiff diff;
+
+        public SchemaTransformationResult(DistributedSchema before, DistributedSchema after, Keyspaces.KeyspacesDiff diff)
+        {
+            this.before = before;
+            this.after = after;
+            this.diff = diff;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("SchemaTransformationResult{%s --> %s, diff=%s}", before.getVersion(), after.getVersion(), diff);
+        }
+    }
 }

src/java/org/apache/cassandra/schema/SchemaTransformations.java

@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Optional;
+
+import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.exceptions.AlreadyExistsException;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
+
+/**
+ * Factory and utility methods to create simple schema transformation.
+ */
+public class SchemaTransformations
+{
+    /**
+     * Creates a schema transformation that adds the provided keyspace.
+     *
+     * @param keyspace       the keyspace to add.
+     * @param ignoreIfExists if {@code true}, the transformation is a no-op if a keyspace of the same name than
+     *                       {@code keyspace} already exists in the schema the transformation is applied on. Otherwise,
+     *                       the transformation throws an {@link AlreadyExistsException} in that case.
+     * @return the created transformation.
+     */
+    public static SchemaTransformation addKeyspace(KeyspaceMetadata keyspace, boolean ignoreIfExists)
+    {
+        return schema ->
+        {
+            KeyspaceMetadata existing = schema.getNullable(keyspace.name);
+            if (existing != null)
+            {
+                if (ignoreIfExists)
+                    return schema;
+
+                throw new AlreadyExistsException(keyspace.name);
+            }
+
+            return schema.withAddedOrUpdated(keyspace);
+        };
+    }
+
+    /**
+     * Creates a schema transformation that adds the provided table.
+     *
+     * @param table          the table to add.
+     * @param ignoreIfExists if {@code true}, the transformation is a no-op if a table of the same name than
+     *                       {@code table} already exists in the schema the transformation is applied on. Otherwise,
+     *                       the transformation throws an {@link AlreadyExistsException} in that case.
+     * @return the created transformation.
+     */
+    public static SchemaTransformation addTable(TableMetadata table, boolean ignoreIfExists)
+    {
+        return schema ->
+        {
+            KeyspaceMetadata keyspace = schema.getNullable(table.keyspace);
+            if (keyspace == null)
+                throw invalidRequest("Keyspace '%s' doesn't exist", table.keyspace);
+
+            if (keyspace.hasTable(table.name))
+            {
+                if (ignoreIfExists)
+                    return schema;
+
+                throw new AlreadyExistsException(table.keyspace, table.name);
+            }
+
+            table.validate();
+
+            return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.with(table)));
+        };
+    }
+
+    public static SchemaTransformation addTypes(Types toAdd, boolean ignoreIfExists)
+    {
+        return schema ->
+        {
+            if (toAdd.isEmpty())
+                return schema;
+
+            String keyspaceName = toAdd.iterator().next().keyspace;
+            KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
+            if (null == keyspace)
+                throw invalidRequest("Keyspace '%s' doesn't exist", keyspaceName);
+
+            Types types = keyspace.types;
+            for (UserType type : toAdd)
+            {
+                if (types.containsType(type.name))
+                {
+                    if (ignoreIfExists)
+                        continue;
+
+                    throw new ConfigurationException("Type " + type + " already exists in " + keyspaceName);
+                }
+
+                types = types.with(type);
+            }
+            return schema.withAddedOrReplaced(keyspace.withSwapped(types));
+        };
+    }
+
+    /**
+     * Creates a schema transformation that adds the provided view.
+     *
+     * @param view           the view to add.
+     * @param ignoreIfExists if {@code true}, the transformation is a no-op if a view of the same name than
+     *                       {@code view} already exists in the schema the transformation is applied on. Otherwise,
+     *                       the transformation throws an {@link AlreadyExistsException} in that case.
+     * @return the created transformation.
+     */
+    public static SchemaTransformation addView(ViewMetadata view, boolean ignoreIfExists)
+    {
+        return schema ->
+        {
+            KeyspaceMetadata keyspace = schema.getNullable(view.keyspace());
+            if (keyspace == null)
+                throw invalidRequest("Cannot add view to non existing keyspace '%s'", view.keyspace());
+
+            if (keyspace.hasView(view.name()))
+            {
+                if (ignoreIfExists)
+                    return schema;
+
+                throw new AlreadyExistsException(view.keyspace(), view.name());
+            }
+
+            return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.views.with(view)));
+        };
+    }
+
+    /**
+     * We have a set of non-local, distributed system keyspaces, e.g. system_traces, system_auth, etc.
+     * (see {@link SchemaConstants#REPLICATED_SYSTEM_KEYSPACE_NAMES}), that need to be created on cluster initialisation,
+     * and later evolved on major upgrades (sometimes minor too). This method compares the current known definitions
+     * of the tables (if the keyspace exists) to the expected, most modern ones expected by the running version of C*.
+     * If any changes have been detected, a schema transformation returned by this method should make cluster's view of
+     * that keyspace aligned with the expected modern definition.
+     *
+     * @param keyspace   the metadata of the keyspace as it should be after application.
+     * @param generation timestamp to use for the table changes in the schema mutation
+     * @return the transformation.
+     */
+    public static SchemaTransformation updateSystemKeyspace(KeyspaceMetadata keyspace, long generation)
+    {
+        return new SchemaTransformation()
+        {
+            @Override
+            public Optional<Long> fixedTimestampMicros()
+            {
+                return Optional.of(generation);
+            }
+
+            @Override
+            public Keyspaces apply(Keyspaces schema)
+            {
+                KeyspaceMetadata updatedKeyspace = keyspace;
+                KeyspaceMetadata curKeyspace = schema.getNullable(keyspace.name);
+                if (curKeyspace != null)
+                {
+                    // If the keyspace already exists, we preserve whatever parameters it has.
+                    updatedKeyspace = updatedKeyspace.withSwapped(curKeyspace.params);
+
+                    for (TableMetadata curTable : curKeyspace.tables)
+                    {
+                        TableMetadata desiredTable = updatedKeyspace.tables.getNullable(curTable.name);
+                        if (desiredTable == null)
+                        {
+                            // preserve exsiting tables which are missing in the new keyspace definition
+                            updatedKeyspace = updatedKeyspace.withSwapped(updatedKeyspace.tables.with(curTable));
+                        }
+                        else
+                        {
+                            updatedKeyspace = updatedKeyspace.withSwapped(updatedKeyspace.tables.without(desiredTable));
+
+                            TableMetadata.Builder updatedBuilder = desiredTable.unbuild();
+
+                            for (ColumnMetadata column : curTable.regularAndStaticColumns())
+                            {
+                                if (!desiredTable.regularAndStaticColumns().contains(column))
+                                    updatedBuilder.addColumn(column);
+                            }
+
+                            updatedKeyspace = updatedKeyspace.withSwapped(updatedKeyspace.tables.with(updatedBuilder.build()));
+                        }
+                    }
+                }
+                return schema.withAddedOrReplaced(updatedKeyspace);
+            }
+        };
+    }
+
+}

src/java/org/apache/cassandra/schema/SchemaUpdateHandler.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.time.Duration;
+
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
+
+/**
+ * Schema update handler is responsible for maintaining the shared schema and synchronizing it with other nodes in
+ * the cluster, which means pushing and pulling changes, as well as tracking the current version in the cluster.
+ * <p/>
+ * The interface has been extracted to abstract out that functionality. It allows for various implementations like
+ * Gossip based (the default), ETCD, offline, etc., and make it easier for mocking in unit tests.
+ */
+public interface SchemaUpdateHandler
+{
+    /**
+     * Starts actively synchronizing schema with the rest of the cluster. It is called in the very beginning of the
+     * node startup. It is not expected to block - to await for the startup completion we have another method
+     * {@link #waitUntilReady(Duration)}.
+     */
+    void start();
+
+    /**
+     * Waits until the schema update handler is ready and returns the result. If the method returns {@code false} it
+     * means that readiness could not be achieved within the specified period of time. The method can be used just to
+     * check if schema is ready by passing {@link Duration#ZERO} as the timeout - in such case it returns immediately.
+     *
+     * @param timeout the maximum time to wait for schema readiness
+     * @return whether readiness is achieved
+     */
+    boolean waitUntilReady(Duration timeout);
+
+    /**
+     * Applies schema transformation in the underlying storage and synchronizes with other nodes.
+     *
+     * @param transformation schema transformation to be performed
+     * @param local          if true, the caller does not require synchronizing schema with other nodes - in practise local is
+     *                       used only in some tests
+     * @return transformation result
+     */
+    SchemaTransformationResult apply(SchemaTransformation transformation, boolean local);
+
+    /**
+     * Resets the schema either by reloading data from the local storage or from the other nodes. Once the schema is
+     * refreshed, the callbacks provided in the factory method are executed, and the updated schema version is announced.
+     *
+     * @param local whether we should reset with locally stored schema or fetch the schema from other nodes
+     * @return transformation result
+     */
+    SchemaTransformationResult reset(boolean local);
+
+    /**
+     * Clears the locally stored schema entirely. After this operation the schema is equal to {@link DistributedSchema#EMPTY}.
+     * The method does not execute any callback. It is indended to reinitialize the schema later using the method
+     * {@link #reset(boolean)}.
+     */
+    void clear();
+}

src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactory.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.function.BiConsumer;
+
+import org.apache.cassandra.schema.SchemaTransformation.SchemaTransformationResult;
+
+public interface SchemaUpdateHandlerFactory
+{
+    /**
+     * A factory which provides the appropriate schema update handler. The actual implementation may be different for
+     * different run modes (client, tool, daemon).
+     *
+     * @param online               whether schema update handler should work online and be aware of the other nodes (when in daemon mode)
+     * @param updateSchemaCallback callback which will be called right after the shared schema is updated
+     */
+    SchemaUpdateHandler getSchemaUpdateHandler(boolean online, BiConsumer<SchemaTransformationResult, Boolean> updateSchemaCallback);
+}

src/java/org/apache/cassandra/schema/SchemaUpdateHandlerFactoryProvider.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import javax.inject.Provider;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Provides the instance of SchemaUpdateHandler factory pointed by {@link #SUH_FACTORY_CLASS_PROPERTY} system property.
+ * If the property is not defined, the default factory {@link DefaultSchemaUpdateHandler} instance is returned.
+ */
+public class SchemaUpdateHandlerFactoryProvider implements Provider<SchemaUpdateHandlerFactory>
+{
+    private static final String SUH_FACTORY_CLASS_PROPERTY = "cassandra.schema.update_handler_factory.class";
+
+    public final static SchemaUpdateHandlerFactoryProvider instance = new SchemaUpdateHandlerFactoryProvider();
+
+    @Override
+    public SchemaUpdateHandlerFactory get()
+    {
+        String suhFactoryClassName = StringUtils.trimToNull(System.getProperty(SUH_FACTORY_CLASS_PROPERTY));
+        if (suhFactoryClassName == null)
+        {
+            return DefaultSchemaUpdateHandlerFactory.instance;
+        }
+        else
+        {
+            Class<SchemaUpdateHandlerFactory> suhFactoryClass = FBUtilities.classForName(suhFactoryClassName, "schema update handler factory");
+            try
+            {
+                return suhFactoryClass.newInstance();
+            }
+            catch (InstantiationException | IllegalAccessException ex)
+            {
+                throw new ConfigurationException(String.format("Failed to initialize schema update handler factory class %s defined in %s system property.",
+                                                               suhFactoryClassName, SUH_FACTORY_CLASS_PROPERTY), ex);
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/schema/TableId.java

@@ -28,6 +28,8 @@ import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * The unique identifier of a table.
  * <p>
@@ -76,7 +78,7 @@ public class TableId
 
     public static TableId unsafeDeterministic(String keyspace, String table)
     {
-        return new TableId(UUID.nameUUIDFromBytes(ArrayUtils.addAll(keyspace.getBytes(), table.getBytes())));
+        return new TableId(UUID.nameUUIDFromBytes(ArrayUtils.addAll(keyspace.getBytes(UTF_8), table.getBytes(UTF_8))));
     }
 
     public String toHexString()

src/java/org/apache/cassandra/schema/TableMetadataRefCache.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Collections;
+import java.util.Map;
+
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Manages the cached {@link TableMetadataRef} objects which holds the references to {@link TableMetadata} objects.
+ * <p>
+ * The purpose of {@link TableMetadataRef} is that the reference to {@link TableMetadataRef} remains unchanged when
+ * the metadata of the table changes. {@link TableMetadata} is immutable, so when it changes, we only switch
+ * the reference inside the existing {@link TableMetadataRef} object.
+ */
+class TableMetadataRefCache
+{
+    public final static TableMetadataRefCache EMPTY = new TableMetadataRefCache(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+
+    // UUID -> mutable metadata ref map. We have to update these in place every time a table changes.
+    private final Map<TableId, TableMetadataRef> metadataRefs;
+
+    // keyspace and table names -> mutable metadata ref map.
+    private final Map<Pair<String, String>, TableMetadataRef> metadataRefsByName;
+
+    // (keyspace name, index name) -> mutable metadata ref map. We have to update these in place every time an index changes.
+    private final Map<Pair<String, String>, TableMetadataRef> indexMetadataRefs;
+
+    public TableMetadataRefCache(Map<TableId, TableMetadataRef> metadataRefs,
+                                 Map<Pair<String, String>, TableMetadataRef> metadataRefsByName,
+                                 Map<Pair<String, String>, TableMetadataRef> indexMetadataRefs)
+    {
+        this.metadataRefs = Collections.unmodifiableMap(metadataRefs);
+        this.metadataRefsByName = Collections.unmodifiableMap(metadataRefsByName);
+        this.indexMetadataRefs = Collections.unmodifiableMap(indexMetadataRefs);
+    }
+
+    /**
+     * Returns cache copy with added the {@link TableMetadataRef} corresponding to the provided keyspace to {@link #metadataRefs} and
+     * {@link #indexMetadataRefs}, assuming the keyspace is new (in the sense of not being tracked by the manager yet).
+     */
+    TableMetadataRefCache withNewRefs(KeyspaceMetadata ksm)
+    {
+        return withUpdatedRefs(ksm.empty(), ksm);
+    }
+
+    /**
+     * Returns cache copy with updated the {@link TableMetadataRef} in {@link #metadataRefs} and {@link #indexMetadataRefs},
+     * for an existing updated keyspace given it's previous and new definition.
+     * <p>
+     * Note that {@link TableMetadataRef} are not duplicated and table metadata is altered in the existing refs.
+     */
+    TableMetadataRefCache withUpdatedRefs(KeyspaceMetadata previous, KeyspaceMetadata updated)
+    {
+        Tables.TablesDiff tablesDiff = Tables.diff(previous.tables, updated.tables);
+        Views.ViewsDiff viewsDiff = Views.diff(previous.views, updated.views);
+
+        MapDifference<String, TableMetadata> indexesDiff = previous.tables.indexesDiff(updated.tables);
+
+        boolean hasCreatedOrDroppedTablesOrViews = tablesDiff.created.size() > 0 || tablesDiff.dropped.size() > 0 || viewsDiff.created.size() > 0 || viewsDiff.dropped.size() > 0;
+        boolean hasCreatedOrDroppedIndexes = !indexesDiff.entriesOnlyOnRight().isEmpty() || !indexesDiff.entriesOnlyOnLeft().isEmpty();
+
+        Map<TableId, TableMetadataRef> metadataRefs = hasCreatedOrDroppedTablesOrViews ? Maps.newHashMap(this.metadataRefs) : this.metadataRefs;
+        Map<Pair<String, String>, TableMetadataRef> metadataRefsByName = hasCreatedOrDroppedTablesOrViews ? Maps.newHashMap(this.metadataRefsByName) : this.metadataRefsByName;
+        Map<Pair<String, String>, TableMetadataRef> indexMetadataRefs = hasCreatedOrDroppedIndexes ? Maps.newHashMap(this.indexMetadataRefs) : this.indexMetadataRefs;
+
+        // clean up after removed entries
+        tablesDiff.dropped.forEach(ref -> removeRef(metadataRefs, metadataRefsByName, ref));
+        viewsDiff.dropped.forEach(view -> removeRef(metadataRefs, metadataRefsByName, view.metadata));
+        indexesDiff.entriesOnlyOnLeft()
+                   .values()
+                   .forEach(indexTable -> indexMetadataRefs.remove(Pair.create(indexTable.keyspace, indexTable.indexName().get())));
+
+        // load up new entries
+        tablesDiff.created.forEach(table -> putRef(metadataRefs, metadataRefsByName, new TableMetadataRef(table)));
+        viewsDiff.created.forEach(view -> putRef(metadataRefs, metadataRefsByName, new TableMetadataRef(view.metadata)));
+        indexesDiff.entriesOnlyOnRight()
+                   .values()
+                   .forEach(indexTable -> indexMetadataRefs.put(Pair.create(indexTable.keyspace, indexTable.indexName().get()), new TableMetadataRef(indexTable)));
+
+        // refresh refs to updated ones
+        tablesDiff.altered.forEach(diff -> metadataRefs.get(diff.after.id).set(diff.after));
+        viewsDiff.altered.forEach(diff -> metadataRefs.get(diff.after.metadata.id).set(diff.after.metadata));
+        indexesDiff.entriesDiffering()
+                   .values()
+                   .stream()
+                   .map(MapDifference.ValueDifference::rightValue)
+                   .forEach(indexTable -> indexMetadataRefs.get(Pair.create(indexTable.keyspace, indexTable.indexName().get())).set(indexTable));
+
+        return new TableMetadataRefCache(metadataRefs, metadataRefsByName, indexMetadataRefs);
+    }
+
+    private void putRef(Map<TableId, TableMetadataRef> metadataRefs,
+                        Map<Pair<String, String>, TableMetadataRef> metadataRefsByName,
+                        TableMetadataRef ref)
+    {
+        metadataRefs.put(ref.id, ref);
+        metadataRefsByName.put(Pair.create(ref.keyspace, ref.name), ref);
+    }
+
+    private void removeRef(Map<TableId, TableMetadataRef> metadataRefs,
+                           Map<Pair<String, String>, TableMetadataRef> metadataRefsByName,
+                           TableMetadata tm)
+    {
+        metadataRefs.remove(tm.id);
+        metadataRefsByName.remove(Pair.create(tm.keyspace, tm.name));
+    }
+
+    /**
+     * Returns cache copy with removed the {@link TableMetadataRef} from {@link #metadataRefs} and {@link #indexMetadataRefs}
+     * for the provided (dropped) keyspace.
+     */
+    TableMetadataRefCache withRemovedRefs(KeyspaceMetadata ksm)
+    {
+        return withUpdatedRefs(ksm, ksm.empty());
+    }
+
+    public TableMetadataRef getTableMetadataRef(TableId id)
+    {
+        return metadataRefs.get(id);
+    }
+
+    public TableMetadataRef getTableMetadataRef(String keyspace, String table)
+    {
+        return metadataRefsByName.get(Pair.create(keyspace, table));
+    }
+
+    public TableMetadataRef getIndexTableMetadataRef(String keyspace, String index)
+    {
+        return indexMetadataRefs.get(Pair.create(keyspace, index));
+    }
+}

src/java/org/apache/cassandra/schema/Types.java

@@ -109,6 +109,11 @@ public final class Types implements Iterable<UserType>
         return Iterables.filter(types.values(), t -> t.referencesUserType(name) && !t.name.equals(name));
     }
 
+    public boolean isEmpty()
+    {
+        return types.isEmpty();
+    }
+
     /**
      * Get the type with the specified name
      *

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -372,8 +372,6 @@ public class CassandraDaemon
             throw new RuntimeException(e);
         }
 
-        SystemKeyspace.finishStartup();
-
         // Clean up system.size_estimates entries left lying around from missed keyspace drops (CASSANDRA-14905)
         StorageService.instance.cleanupSizeEstimates();
 
@@ -441,7 +439,7 @@ public class CassandraDaemon
             logger.debug("Completed submission of build tasks for any materialized views defined at startup");
         };
 
-        ScheduledExecutors.optionalTasks.schedule(viewRebuild, StorageService.RING_DELAY, TimeUnit.MILLISECONDS);
+        ScheduledExecutors.optionalTasks.schedule(viewRebuild, StorageService.RING_DELAY_MILLIS, TimeUnit.MILLISECONDS);
 
         if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
             Gossiper.waitToSettle();

src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java

@@ -18,7 +18,7 @@
 
 package org.apache.cassandra.service;
 
-import java.util.List;
+import java.util.Collection;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -53,7 +53,7 @@ public class PendingRangeCalculatorService
     private final AtLeastOnceTrigger update = executor.atLeastOnceTrigger(() -> {
         PendingRangeCalculatorServiceDiagnostics.taskStarted(1);
         long start = currentTimeMillis();
-        List<String> keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
+        Collection<String> keyspaces = Schema.instance.getNonLocalStrategyKeyspaces().names();
         for (String keyspaceName : keyspaces)
             calculatePendingRanges(Keyspace.open(keyspaceName).getReplicationStrategy(), keyspaceName);
         if (logger.isTraceEnabled())

src/java/org/apache/cassandra/service/StorageService.java

@@ -71,6 +71,7 @@ import org.apache.cassandra.fql.FullQueryLoggerOptions;
 import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -126,11 +127,10 @@ import org.apache.cassandra.repair.*;
 import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
 import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.MigrationCoordinator;
-import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaTransformations;
 import org.apache.cassandra.schema.SystemDistributedKeyspace;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
@@ -172,7 +172,6 @@ import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName;
 import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily;
 import static org.apache.cassandra.net.NoPayload.noPayload;
 import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ;
-import static org.apache.cassandra.schema.MigrationManager.evolveSystemKeyspace;
 import static org.apache.cassandra.service.ActiveRepairService.*;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@@ -190,7 +189,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     private static final Logger logger = LoggerFactory.getLogger(StorageService.class);
 
     public static final int INDEFINITE = -1;
-    public static final int RING_DELAY = getRingDelay(); // delay after which we assume ring has stablized
+    public static final int RING_DELAY_MILLIS = getRingDelay(); // delay after which we assume ring has stablized
     public static final int SCHEMA_DELAY_MILLIS = getSchemaDelay();
 
     private static final boolean REQUIRE_SCHEMAS = !BOOTSTRAP_SKIP_SCHEMA_CHECK.getBoolean();
@@ -321,7 +320,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     /* the probability for tracing any particular request, 0 disables tracing and 1 enables for all */
     private double traceProbability = 0.0;
 
-    private static enum Mode { STARTING, NORMAL, JOINING, LEAVING, DECOMMISSIONED, MOVING, DRAINING, DRAINED }
+    private enum Mode { STARTING, NORMAL, JOINING, LEAVING, DECOMMISSIONED, MOVING, DRAINING, DRAINED }
     private volatile Mode operationMode = Mode.STARTING;
 
     /* Used for tracking drain progress */
@@ -784,10 +783,15 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public synchronized void initServer() throws ConfigurationException
     {
-        initServer(RING_DELAY);
+        initServer(SCHEMA_DELAY_MILLIS, RING_DELAY_MILLIS);
     }
 
-    public synchronized void initServer(int delay) throws ConfigurationException
+    public synchronized void initServer(int schemaAndRingDelayMillis) throws ConfigurationException
+    {
+        initServer(schemaAndRingDelayMillis, RING_DELAY_MILLIS);
+    }
+
+    public synchronized void initServer(int schemaTimeoutMillis, int ringTimeoutMillis) throws ConfigurationException
     {
         logger.info("Cassandra version: {}", FBUtilities.getReleaseVersionString());
         logger.info("CQL version: {}", QueryProcessor.CQL_VERSION);
@@ -850,7 +854,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         if (joinRing)
         {
-            joinTokenRing(delay);
+            joinTokenRing(schemaTimeoutMillis, ringTimeoutMillis);
         }
         else
         {
@@ -906,13 +910,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         tokenMetadata.updateHostIds(hostIdToEndpointMap);
     }
 
-    private boolean isReplacing()
+    public boolean isReplacing()
     {
+        if (replacing)
+            return true;
+
         if (System.getProperty("cassandra.replace_address_first_boot", null) != null && SystemKeyspace.bootstrapComplete())
         {
             logger.info("Replace address on first boot requested; this node is already bootstrapped");
             return false;
         }
+
         return DatabaseDescriptor.getReplaceAddress() != null;
     }
 
@@ -939,7 +947,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     private void prepareToJoin() throws ConfigurationException
     {
-        MigrationCoordinator.instance.start();
         if (!joined)
         {
             Map<ApplicationState, VersionedValue> appStates = new EnumMap<>(ApplicationState.class);
@@ -983,7 +990,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                     appStates.put(ApplicationState.STATUS_WITH_PORT, valueFactory.hibernate(true));
                     appStates.put(ApplicationState.STATUS, valueFactory.hibernate(true));
                 }
-                MigrationCoordinator.instance.removeAndIgnoreEndpoint(DatabaseDescriptor.getReplaceAddress());
             }
             else
             {
@@ -1029,8 +1035,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
             // gossip snitch infos (local DC and rack)
             gossipSnitchInfo();
-            // gossip Schema.emptyVersion forcing immediate check for schema updates (see MigrationManager#maybeScheduleSchemaPull)
-            Schema.instance.updateVersionAndAnnounce(); // Ensure we know our own actual Schema UUID in preparation for updates
+            Schema.instance.startSync();
             LoadBroadcaster.instance.startBroadcasting();
             HintsService.instance.startDispatch();
             BatchlogManager.instance.start();
@@ -1038,47 +1043,28 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
     }
 
-    public void waitForSchema(long delay)
+    public void waitForSchema(long schemaTimeoutMillis, long ringTimeoutMillis)
     {
-        // first sleep the delay to make sure we see all our peers
-        for (long i = 0; i < delay; i += 1000)
-        {
-            // if we see schema, we can proceed to the next check directly
-            if (!Schema.instance.isEmpty())
-            {
-                logger.debug("current schema version: {}", Schema.instance.getVersion());
-                break;
-            }
+        Instant deadline = FBUtilities.now().plus(java.time.Duration.ofMillis(ringTimeoutMillis));
+
+        while (Schema.instance.isEmpty() && FBUtilities.now().isBefore(deadline))
             Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
-        }
 
-        boolean schemasReceived = MigrationCoordinator.instance.awaitSchemaRequests(SCHEMA_DELAY_MILLIS);
-
-        if (schemasReceived)
-            return;
-
-        logger.warn(String.format("There are nodes in the cluster with a different schema version than us we did not merged schemas from, " +
-                                  "our version : (%s), outstanding versions -> endpoints : %s. Use -Dcassandra.skip_schema_check=true " +
-                                  "to ignore this, -Dcassandra.skip_schema_check_for_endpoints=<ep1[,epN]> to skip specific endpoints," +
-                                  "or -Dcassandra.skip_schema_check_for_versions=<ver1[,verN]> to skip specific schema versions",
-                                  Schema.instance.getVersion(),
-                                  MigrationCoordinator.instance.outstandingVersions()));
-
-        if (REQUIRE_SCHEMAS)
-            throw new RuntimeException("Didn't receive schemas for all known versions within the timeout. " +
-                                       "Use -Dcassandra.skip_schema_check=true to skip this check.");
+        if (!Schema.instance.waitUntilReady(java.time.Duration.ofMillis(schemaTimeoutMillis)))
+            throw new IllegalStateException("Could not achieve schema readiness in " + java.time.Duration.ofMillis(schemaTimeoutMillis));
     }
 
-    private void joinTokenRing(long schemaTimeoutMillis) throws ConfigurationException
+    private void joinTokenRing(long schemaTimeoutMillis, long ringTimeoutMillis) throws ConfigurationException
     {
-        joinTokenRing(!isSurveyMode, shouldBootstrap(), schemaTimeoutMillis, INDEFINITE);
+        joinTokenRing(!isSurveyMode, shouldBootstrap(), schemaTimeoutMillis, INDEFINITE, ringTimeoutMillis);
     }
 
     @VisibleForTesting
     public void joinTokenRing(boolean finishJoiningRing,
                               boolean shouldBootstrap,
                               long schemaTimeoutMillis,
-                              long bootstrapTimeoutMillis) throws ConfigurationException
+                              long bootstrapTimeoutMillis,
+                              long ringTimeoutMillis) throws ConfigurationException
     {
         joined = true;
 
@@ -1109,7 +1095,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         if (shouldBootstrap)
         {
-            current.addAll(prepareForBootstrap(schemaTimeoutMillis));
+            current.addAll(prepareForBootstrap(schemaTimeoutMillis, ringTimeoutMillis));
             dataAvailable = bootstrap(bootstrapTokens, bootstrapTimeoutMillis);
         }
         else
@@ -1117,7 +1103,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             bootstrapTokens = SystemKeyspace.getSavedTokens();
             if (bootstrapTokens.isEmpty())
             {
-                bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis);
+                bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis, ringTimeoutMillis);
             }
             else
             {
@@ -1188,7 +1174,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             logger.info("Joining ring by operator request");
             try
             {
-                joinTokenRing(0);
+                joinTokenRing(SCHEMA_DELAY_MILLIS, 0);
                 doAuthSetup(false);
             }
             catch (ConfigurationException e)
@@ -1223,7 +1209,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     private void executePreJoinTasks(boolean bootstrap)
     {
         StreamSupport.stream(ColumnFamilyStore.all().spliterator(), false)
-                .filter(cfs -> Schema.instance.getUserKeyspaces().contains(cfs.keyspace.getName()))
+                .filter(cfs -> Schema.instance.getUserKeyspaces().names().contains(cfs.keyspace.getName()))
                 .forEach(cfs -> cfs.indexManager.executePreJoinTasksBlocking(bootstrap));
     }
 
@@ -1246,8 +1232,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         {
             if (setUpSchema)
             {
-                Optional<Mutation> mutation = evolveSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION);
-                mutation.ifPresent(value -> FBUtilities.waitOnFuture(MigrationManager.announceWithoutPush(Collections.singleton(value))));
+                Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION));
             }
 
             DatabaseDescriptor.getRoleManager().setup();
@@ -1275,14 +1260,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     @VisibleForTesting
     public void setUpDistributedSystemKeyspaces()
     {
-        Collection<Mutation> changes = new ArrayList<>(3);
-
-        evolveSystemKeyspace(            TraceKeyspace.metadata(),             TraceKeyspace.GENERATION).ifPresent(changes::add);
-        evolveSystemKeyspace(SystemDistributedKeyspace.metadata(), SystemDistributedKeyspace.GENERATION).ifPresent(changes::add);
-        evolveSystemKeyspace(             AuthKeyspace.metadata(),              AuthKeyspace.GENERATION).ifPresent(changes::add);
-
-        if (!changes.isEmpty())
-            FBUtilities.waitOnFuture(MigrationManager.announceWithoutPush(changes));
+        Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(TraceKeyspace.metadata(), TraceKeyspace.GENERATION));
+        Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(SystemDistributedKeyspace.metadata(), SystemDistributedKeyspace.GENERATION));
+        Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION));
     }
 
     public boolean isJoined()
@@ -1331,7 +1311,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
             if (keyspace == null)
             {
-                for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+                for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
                     streamer.addRanges(keyspaceName, getLocalReplicas(keyspaceName));
             }
             else if (tokens == null)
@@ -1751,7 +1731,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     }
 
     @VisibleForTesting
-    public Collection<InetAddressAndPort> prepareForBootstrap(long schemaDelay)
+    public Collection<InetAddressAndPort> prepareForBootstrap(long schemaTimeoutMillis, long ringTimeoutMillis)
     {
         Set<InetAddressAndPort> collisions = new HashSet<>();
         if (SystemKeyspace.bootstrapInProgress())
@@ -1759,7 +1739,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         else
             SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.IN_PROGRESS);
         setMode(Mode.JOINING, "waiting for ring information", true);
-        waitForSchema(schemaDelay);
+        waitForSchema(schemaTimeoutMillis, ringTimeoutMillis);
         setMode(Mode.JOINING, "schema complete, ready to bootstrap", true);
         setMode(Mode.JOINING, "waiting for pending range calculation", true);
         PendingRangeCalculatorService.instance.blockUntilFinished();
@@ -1789,7 +1769,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 throw new UnsupportedOperationException(s);
             }
             setMode(Mode.JOINING, "getting bootstrap token", true);
-            bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), schemaDelay);
+            bootstrapTokens = BootStrapper.getBootstrapTokens(tokenMetadata, FBUtilities.getBroadcastAddressAndPort(), schemaTimeoutMillis, ringTimeoutMillis);
         }
         else
         {
@@ -1812,7 +1792,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                     InetAddressAndPort existing = tokenMetadata.getEndpoint(token);
                     if (existing != null)
                     {
-                        long nanoDelay = schemaDelay * 1000000L;
+                        long nanoDelay = ringTimeoutMillis * 1000000L;
                         if (Gossiper.instance.getEndpointStateForEndpoint(existing).getUpdateTimestamp() > (nanoTime() - nanoDelay))
                             throw new UnsupportedOperationException("Cannot replace a live node... ");
                         collisions.add(existing);
@@ -1827,7 +1807,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             {
                 try
                 {
-                    Thread.sleep(RING_DELAY);
+                    Thread.sleep(RING_DELAY_MILLIS);
                 }
                 catch (InterruptedException e)
                 {
@@ -1867,8 +1847,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                                                             valueFactory.bootReplacing(DatabaseDescriptor.getReplaceAddress().getAddress()) :
                                                             valueFactory.bootstrapping(tokens)));
             Gossiper.instance.addLocalApplicationStates(states);
-            setMode(Mode.JOINING, "sleeping " + RING_DELAY + " ms for pending range setup", true);
-            Uninterruptibles.sleepUninterruptibly(RING_DELAY, MILLISECONDS);
+            setMode(Mode.JOINING, "sleeping " + RING_DELAY_MILLIS + " ms for pending range setup", true);
+            Uninterruptibles.sleepUninterruptibly(RING_DELAY_MILLIS, MILLISECONDS);
         }
         else
         {
@@ -1938,7 +1918,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
      * All MVs have been created during bootstrap, so mark them as built
      */
     private void markViewsAsBuilt() {
-        for (String keyspace : Schema.instance.getUserKeyspaces())
+        for (String keyspace : Schema.instance.getUserKeyspaces().names())
         {
             for (ViewMetadata view: Schema.instance.getKeyspaceMetadata(keyspace).views)
                 SystemKeyspace.finishViewBuildStatus(view.keyspace(), view.name());
@@ -2147,7 +2127,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // some people just want to get a visual representation of things. Allow null and set it to the first
         // non-system keyspace.
         if (keyspace == null)
-            keyspace = Schema.instance.getNonLocalStrategyKeyspaces().get(0);
+            keyspace = Schema.instance.getNonLocalStrategyKeyspaces().iterator().next().name;
 
         Map<List<String>, List<String>> map = new HashMap<>();
         for (Map.Entry<Range<Token>, EndpointsForRange> entry : tokenMetadata.getPendingRangesMM(keyspace).asMap().entrySet())
@@ -2201,7 +2181,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // some people just want to get a visual representation of things. Allow null and set it to the first
         // non-system keyspace.
         if (keyspace == null)
-            keyspace = Schema.instance.getNonLocalStrategyKeyspaces().get(0);
+            keyspace = Schema.instance.getNonLocalStrategyKeyspaces().iterator().next().name;
 
         List<Range<Token>> ranges = getAllRanges(sortedTokens);
         return constructRangeToEndpointMap(keyspace, ranges);
@@ -2501,7 +2481,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                         break;
                     case SCHEMA:
                         SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(value.value));
-                        MigrationCoordinator.instance.reportEndpointVersion(endpoint, UUID.fromString(value.value));
                         break;
                     case HOST_ID:
                         SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(value.value));
@@ -3131,7 +3110,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     private void removeEndpoint(InetAddressAndPort endpoint)
     {
         Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.removeEndpoint(endpoint));
-        MigrationCoordinator.instance.removeAndIgnoreEndpoint(endpoint);
         SystemKeyspace.removeEndpoint(endpoint);
     }
 
@@ -3293,7 +3271,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
 
-        for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+        for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
         {
             logger.debug("Restoring replica count for keyspace {}", keyspaceName);
             EndpointsByReplica changedReplicas = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
@@ -3445,13 +3423,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             statusValue = epState.getApplicationState(statusState);
         }
         if (statusValue != null)
-            onChange(endpoint, statusState, statusValue);
+            Gossiper.instance.doOnChangeNotifications(endpoint, statusState, statusValue);
 
         for (Map.Entry<ApplicationState, VersionedValue> entry : epState.states())
         {
             if (entry.getKey() == ApplicationState.STATUS_WITH_PORT || entry.getKey() == ApplicationState.STATUS)
                 continue;
-            onChange(endpoint, entry.getKey(), entry.getValue());
+            Gossiper.instance.doOnChangeNotifications(endpoint, entry.getKey(), entry.getValue());
         }
     }
 
@@ -4389,8 +4367,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         List<Future<?>> futures = new ArrayList<>();
 
-        List<String> keyspaces = Schema.instance.getNonLocalStrategyKeyspaces() ;
-        for (String ksName : keyspaces)
+        Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
+        for (String ksName : keyspaces.names())
         {
             if (SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(ksName))
                 continue;
@@ -4722,7 +4700,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             if (operationMode != Mode.LEAVING) // If we're already decommissioning there is no point checking RF/pending ranges
             {
                 int rf, numNodes;
-                for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+                for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
                 {
                     if (!force)
                     {
@@ -4751,7 +4729,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             }
 
             startLeaving();
-            long timeout = Math.max(RING_DELAY, BatchlogManager.instance.getBatchlogTimeout());
+            long timeout = Math.max(RING_DELAY_MILLIS, BatchlogManager.instance.getBatchlogTimeout());
             setMode(Mode.LEAVING, "sleeping " + timeout + " ms for batch processing and pending range setup", true);
             Thread.sleep(timeout);
 
@@ -4801,7 +4779,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.left(getLocalTokens(),Gossiper.computeExpireTime()));
         Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.left(getLocalTokens(),Gossiper.computeExpireTime()));
-        int delay = Math.max(RING_DELAY, Gossiper.intervalInMillis * 2);
+        int delay = Math.max(RING_DELAY_MILLIS, Gossiper.intervalInMillis * 2);
         logger.info("Announcing that I have left the ring for {}ms", delay);
         Uninterruptibles.sleepUninterruptibly(delay, MILLISECONDS);
     }
@@ -4810,7 +4788,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     {
         Map<String, EndpointsByReplica> rangesToStream = new HashMap<>();
 
-        for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+        for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
         {
             EndpointsByReplica rangesMM = getChangedReplicasForLeaving(keyspaceName, FBUtilities.getBroadcastAddressAndPort(), tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
 
@@ -4924,7 +4902,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             throw new UnsupportedOperationException("This node has more than one token and cannot be moved thusly.");
         }
 
-        List<String> keyspacesToProcess = Schema.instance.getNonLocalStrategyKeyspaces();
+        List<String> keyspacesToProcess = ImmutableList.copyOf(Schema.instance.getNonLocalStrategyKeyspaces().names());
 
         PendingRangeCalculatorService.instance.blockUntilFinished();
         // checking if data is moving to this node
@@ -4939,8 +4917,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.moving(newToken));
         setMode(Mode.MOVING, String.format("Moving %s from %s to %s.", localAddress, getLocalTokens().iterator().next(), newToken), true);
 
-        setMode(Mode.MOVING, String.format("Sleeping %s ms before start streaming/fetching ranges", RING_DELAY), true);
-        Uninterruptibles.sleepUninterruptibly(RING_DELAY, MILLISECONDS);
+        setMode(Mode.MOVING, String.format("Sleeping %s ms before start streaming/fetching ranges", RING_DELAY_MILLIS), true);
+        Uninterruptibles.sleepUninterruptibly(RING_DELAY_MILLIS, MILLISECONDS);
 
         RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess, tokenMetadata);
         relocator.calculateToFromStreams();
@@ -5071,7 +5049,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         Collection<Token> tokens = tokenMetadata.getTokens(endpoint);
 
         // Find the endpoints that are going to become responsible for data
-        for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces())
+        for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
         {
             // if the replication factor is 1 the data is lost so we shouldn't wait for confirmation
             if (Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor().allReplicas == 1)
@@ -5507,11 +5485,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
         else
         {
-            List<String> userKeyspaces = Schema.instance.getUserKeyspaces();
+            Collection<String> userKeyspaces = Schema.instance.getUserKeyspaces().names();
 
             if (userKeyspaces.size() > 0)
             {
-                keyspace = userKeyspaces.get(0);
+                keyspace = userKeyspaces.iterator().next();
                 AbstractReplicationStrategy replicationStrategy = Schema.instance.getKeyspaceInstance(keyspace).getReplicationStrategy();
                 for (String keyspaceName : userKeyspaces)
                 {
@@ -5578,19 +5556,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public List<String> getKeyspaces()
     {
-        List<String> keyspaceNamesList = new ArrayList<>(Schema.instance.getKeyspaces());
-        return Collections.unmodifiableList(keyspaceNamesList);
+        return Lists.newArrayList(Schema.instance.distributedAndLocalKeyspaces().names());
     }
 
     public List<String> getNonSystemKeyspaces()
     {
-        List<String> nonKeyspaceNamesList = new ArrayList<>(Schema.instance.getNonSystemKeyspaces());
-        return Collections.unmodifiableList(nonKeyspaceNamesList);
+        return Lists.newArrayList(Schema.instance.distributedKeyspaces().names());
     }
 
     public List<String> getNonLocalStrategyKeyspaces()
     {
-        return Collections.unmodifiableList(Schema.instance.getNonLocalStrategyKeyspaces());
+        return Lists.newArrayList(Schema.instance.getNonLocalStrategyKeyspaces().names());
     }
 
     public Map<String, String> getViewBuildStatuses(String keyspace, String view, boolean withPort)
@@ -5911,7 +5887,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public void resetLocalSchema() throws IOException
     {
-        MigrationManager.resetLocalSchema();
+        Schema.instance.resetLocalSchema();
     }
 
     public void reloadLocalSchema()
@@ -6299,7 +6275,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     @Override
     public Map<String, Set<InetAddress>> getOutstandingSchemaVersions()
     {
-        Map<UUID, Set<InetAddressAndPort>> outstanding = MigrationCoordinator.instance.outstandingVersions();
+        Map<UUID, Set<InetAddressAndPort>> outstanding = Schema.instance.getOutstandingSchemaVersions();
         return outstanding.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(),
                                                                         e -> e.getValue().stream().map(InetSocketAddress::getAddress).collect(Collectors.toSet())));
     }
@@ -6307,7 +6283,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     @Override
     public Map<String, Set<String>> getOutstandingSchemaVersionsWithPort()
     {
-        Map<UUID, Set<InetAddressAndPort>> outstanding = MigrationCoordinator.instance.outstandingVersions();
+        Map<UUID, Set<InetAddressAndPort>> outstanding = Schema.instance.getOutstandingSchemaVersions();
         return outstanding.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(),
                                                                         e -> e.getValue().stream().map(Object::toString).collect(Collectors.toSet())));
     }
@@ -6686,7 +6662,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         logger.info("StorageService#clearPaxosRepairs called via jmx");
         PaxosTableRepairs.clearRepairs();
     }
-    
+
     public void setSkipPaxosRepairCompatibilityCheck(boolean v)
     {
         PaxosRepair.setSkipPaxosRepairCompatibilityCheck(v);

src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java

@@ -60,7 +60,7 @@ public class SSTableExpiredBlockers
 
         String keyspace = args[args.length - 2];
         String columnfamily = args[args.length - 1];
-        Schema.instance.loadFromDisk(false);
+        Schema.instance.loadFromDisk();
 
         TableMetadata metadata = Schema.instance.validateTable(keyspace, columnfamily);
 

src/java/org/apache/cassandra/tools/SSTableLevelResetter.java

@@ -65,7 +65,7 @@ public class SSTableLevelResetter
         try
         {
             // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
+            Schema.instance.loadFromDisk();
 
             String keyspaceName = args[1];
             String columnfamily = args[2];

src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java

@@ -91,7 +91,7 @@ public class SSTableOfflineRelevel
         boolean dryRun = args[0].equals("--dry-run");
         String keyspace = args[args.length - 2];
         String columnfamily = args[args.length - 1];
-        Schema.instance.loadFromDisk(false);
+        Schema.instance.loadFromDisk();
 
         if (Schema.instance.getTableMetadataRef(keyspace, columnfamily) == null)
             throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",

src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java

@@ -48,7 +48,7 @@ public class StandaloneSSTableUtil
         {
             // load keyspace descriptions.
             Util.initDatabaseDescriptor();
-            Schema.instance.loadFromDisk(false);
+            Schema.instance.loadFromDisk();
 
             TableMetadata metadata = Schema.instance.getTableMetadata(options.keyspaceName, options.cfName);
             if (metadata == null)

src/java/org/apache/cassandra/tools/StandaloneScrubber.java

@@ -87,7 +87,7 @@ public class StandaloneScrubber
         try
         {
             // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
+            Schema.instance.loadFromDisk();
 
             if (Schema.instance.getKeyspaceMetadata(options.keyspaceName) == null)
                 throw new IllegalArgumentException(String.format("Unknown keyspace %s", options.keyspaceName));

src/java/org/apache/cassandra/tools/StandaloneSplitter.java

@@ -61,7 +61,7 @@ public class StandaloneSplitter
         try
         {
             // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
+            Schema.instance.loadFromDisk();
 
             String ksName = null;
             String cfName = null;

src/java/org/apache/cassandra/tools/StandaloneUpgrader.java

@@ -57,7 +57,7 @@ public class StandaloneUpgrader
         try
         {
             // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
+            Schema.instance.loadFromDisk();
 
             if (Schema.instance.getTableMetadataRef(options.keyspace, options.cf) == null)
                 throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",

src/java/org/apache/cassandra/tools/StandaloneVerifier.java

@@ -74,7 +74,7 @@ public class StandaloneVerifier
         try
         {
             // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
+            Schema.instance.loadFromDisk();
 
             boolean hasFailed = false;
 

src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java

@@ -457,7 +457,7 @@ public class TableStatsHolder implements StatsHolder
             return filter.get(keyspace) != null || ignoreMode;
         }
 
-        public void verifyKeyspaces(List<String> keyspaces)
+        public void verifyKeyspaces(Collection<String> keyspaces)
         {
             for (String ks : verifier.keySet())
                 if (!keyspaces.contains(ks))

src/java/org/apache/cassandra/transport/Server.java

@@ -41,10 +41,15 @@ import io.netty.util.internal.logging.Slf4JLoggerFactory;
 import org.apache.cassandra.auth.AuthenticatedUser;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.cql3.functions.UDAggregate;
+import org.apache.cassandra.cql3.functions.UDFunction;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaChangeListener;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.*;
 import org.apache.cassandra.transport.messages.EventMessage;
 import org.apache.cassandra.utils.FBUtilities;
@@ -354,7 +359,7 @@ public class Server implements CassandraDaemon.Server
         }
     }
 
-    public static class EventNotifier extends SchemaChangeListener implements IEndpointLifecycleSubscriber
+    public static class EventNotifier implements SchemaChangeListener, IEndpointLifecycleSubscriber
     {
         private ConnectionTracker connectionTracker;
 
@@ -407,6 +412,7 @@ public class Server implements CassandraDaemon.Server
             connectionTracker.send(event);
         }
 
+        @Override
         public void onJoinCluster(InetAddressAndPort endpoint)
         {
             if (!StorageService.instance.isRpcReady(endpoint))
@@ -415,16 +421,19 @@ public class Server implements CassandraDaemon.Server
                 onTopologyChange(endpoint, Event.TopologyChange.newNode(getNativeAddress(endpoint)));
         }
 
+        @Override
         public void onLeaveCluster(InetAddressAndPort endpoint)
         {
             onTopologyChange(endpoint, Event.TopologyChange.removedNode(getNativeAddress(endpoint)));
         }
 
+        @Override
         public void onMove(InetAddressAndPort endpoint)
         {
             onTopologyChange(endpoint, Event.TopologyChange.movedNode(getNativeAddress(endpoint)));
         }
 
+        @Override
         public void onUp(InetAddressAndPort endpoint)
         {
             if (endpointsPendingJoinedNotification.remove(endpoint))
@@ -433,6 +442,7 @@ public class Server implements CassandraDaemon.Server
             onStatusChange(endpoint, Event.StatusChange.nodeUp(getNativeAddress(endpoint)));
         }
 
+        @Override
         public void onDown(InetAddressAndPort endpoint)
         {
             onStatusChange(endpoint, Event.StatusChange.nodeDown(getNativeAddress(endpoint)));
@@ -466,85 +476,100 @@ public class Server implements CassandraDaemon.Server
             }
         }
 
-        public void onCreateKeyspace(String ksName)
+        @Override
+        public void onCreateKeyspace(KeyspaceMetadata keyspace)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, ksName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, keyspace.name));
         }
 
-        public void onCreateTable(String ksName, String cfName)
+        @Override
+        public void onCreateTable(TableMetadata table)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, ksName, cfName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, table.keyspace, table.name));
         }
 
-        public void onCreateType(String ksName, String typeName)
+        @Override
+        public void onCreateType(UserType type)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TYPE, ksName, typeName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TYPE, type.keyspace, type.getNameAsString()));
         }
 
-        public void onCreateFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onCreateFunction(UDFunction function)
         {
             send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.FUNCTION,
-                                        ksName, functionName, AbstractType.asCQLTypeStringList(argTypes)));
+                                        function.name().keyspace, function.name().name, AbstractType.asCQLTypeStringList(function.argTypes())));
         }
 
-        public void onCreateAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onCreateAggregate(UDAggregate aggregate)
         {
             send(new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.AGGREGATE,
-                                        ksName, aggregateName, AbstractType.asCQLTypeStringList(argTypes)));
+                                        aggregate.name().keyspace, aggregate.name().name, AbstractType.asCQLTypeStringList(aggregate.argTypes())));
         }
 
-        public void onAlterKeyspace(String ksName)
+        @Override
+        public void onAlterKeyspace(KeyspaceMetadata before, KeyspaceMetadata after)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, ksName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, after.name));
         }
 
-        public void onAlterTable(String ksName, String cfName, boolean affectsStatements)
+        @Override
+        public void onAlterTable(TableMetadata before, TableMetadata after, boolean affectsStatements)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, ksName, cfName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, after.keyspace, after.name));
         }
 
-        public void onAlterType(String ksName, String typeName)
+        @Override
+        public void onAlterType(UserType before, UserType after)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TYPE, ksName, typeName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TYPE, after.keyspace, after.getNameAsString()));
         }
 
-        public void onAlterFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onAlterFunction(UDFunction before, UDFunction after)
         {
             send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.FUNCTION,
-                                        ksName, functionName, AbstractType.asCQLTypeStringList(argTypes)));
+                                        after.name().keyspace, after.name().name, AbstractType.asCQLTypeStringList(after.argTypes())));
         }
 
-        public void onAlterAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onAlterAggregate(UDAggregate before, UDAggregate after)
         {
             send(new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.AGGREGATE,
-                                        ksName, aggregateName, AbstractType.asCQLTypeStringList(argTypes)));
+                                        after.name().keyspace, after.name().name, AbstractType.asCQLTypeStringList(after.argTypes())));
         }
 
-        public void onDropKeyspace(String ksName)
+        @Override
+        public void onDropKeyspace(KeyspaceMetadata keyspace)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, ksName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, keyspace.name));
         }
 
-        public void onDropTable(String ksName, String cfName)
+        @Override
+        public void onDropTable(TableMetadata table)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TABLE, ksName, cfName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TABLE, table.keyspace, table.name));
         }
 
-        public void onDropType(String ksName, String typeName)
+        @Override
+        public void onDropType(UserType type)
         {
-            send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TYPE, ksName, typeName));
+            send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TYPE, type.keyspace, type.getNameAsString()));
         }
 
-        public void onDropFunction(String ksName, String functionName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onDropFunction(UDFunction function)
         {
             send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.FUNCTION,
-                                        ksName, functionName, AbstractType.asCQLTypeStringList(argTypes)));
+                                        function.name().keyspace, function.name().name, AbstractType.asCQLTypeStringList(function.argTypes())));
         }
 
-        public void onDropAggregate(String ksName, String aggregateName, List<AbstractType<?>> argTypes)
+        @Override
+        public void onDropAggregate(UDAggregate aggregate)
         {
             send(new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.AGGREGATE,
-                                        ksName, aggregateName, AbstractType.asCQLTypeStringList(argTypes)));
+                                        aggregate.name().keyspace, aggregate.name().name, AbstractType.asCQLTypeStringList(aggregate.argTypes())));
         }
     }
 }

src/java/org/apache/cassandra/utils/Collectors3.java

@@ -18,22 +18,24 @@
 
 package org.apache.cassandra.utils;
 
-import java.util.List;
-import java.util.Set;
+import java.util.Map;
+import java.util.function.Function;
 import java.util.stream.Collector;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 
 /**
  * Some extra Collector implementations.
- *
+ * <p>
  * Named Collectors3 just in case Guava ever makes a Collectors2
  */
 public class Collectors3
 {
-    private static final Collector.Characteristics[] LIST_CHARACTERISTICS = new Collector.Characteristics[] { };
-    public static <T>  Collector<T, ?, List<T>> toImmutableList()
+    private static final Collector.Characteristics[] LIST_CHARACTERISTICS = new Collector.Characteristics[]{};
+
+    public static <T> Collector<T, ?, ImmutableList<T>> toImmutableList()
     {
         return Collector.of(ImmutableList.Builder<T>::new,
                             ImmutableList.Builder<T>::add,
@@ -42,8 +44,9 @@ public class Collectors3
                             LIST_CHARACTERISTICS);
     }
 
-    private static final Collector.Characteristics[] SET_CHARACTERISTICS = new Collector.Characteristics[] { Collector.Characteristics.UNORDERED };
-    public static <T>  Collector<T, ?, Set<T>> toImmutableSet()
+    private static final Collector.Characteristics[] SET_CHARACTERISTICS = new Collector.Characteristics[]{ Collector.Characteristics.UNORDERED };
+
+    public static <T> Collector<T, ?, ImmutableSet<T>> toImmutableSet()
     {
         return Collector.of(ImmutableSet.Builder<T>::new,
                             ImmutableSet.Builder<T>::add,
@@ -51,4 +54,25 @@ public class Collectors3
                             ImmutableSet.Builder<T>::build,
                             SET_CHARACTERISTICS);
     }
+
+    private static final Collector.Characteristics[] MAP_CHARACTERISTICS = new Collector.Characteristics[]{ Collector.Characteristics.UNORDERED };
+
+    public static <K, V> Collector<Map.Entry<K, V>, ?, ImmutableMap<K, V>> toImmutableMap()
+    {
+        return Collector.of(ImmutableMap.Builder<K, V>::new,
+                            ImmutableMap.Builder<K, V>::put,
+                            (l, r) -> l.putAll(r.build()),
+                            ImmutableMap.Builder<K, V>::build,
+                            MAP_CHARACTERISTICS);
+    }
+
+    public static <T, K, V> Collector<T, ?, ImmutableMap<K, V>> toImmutableMap(Function<? super T, ? extends K> keyMapper, Function<? super T, ? extends V> valueMapper)
+    {
+        return Collector.of(ImmutableMap.Builder<K, V>::new,
+                            (b, t) -> b.put(keyMapper.apply(t), valueMapper.apply(t)),
+                            (l, r) -> l.putAll(r.build()),
+                            ImmutableMap.Builder::build,
+                            MAP_CHARACTERISTICS);
+    }
+
 }

src/java/org/apache/cassandra/utils/FBUtilities.java

@@ -512,7 +512,7 @@ public class FBUtilities
         }
         catch (ExecutionException ee)
         {
-            throw new RuntimeException(ee);
+            throw Throwables.cleaned(ee);
         }
         catch (InterruptedException ie)
         {

test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java

@@ -45,7 +45,7 @@ import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.MigrationCoordinator;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -227,8 +227,8 @@ public class GossipHelper
             pullTo.acceptsOnInstance((InetSocketAddress pullFrom) -> {
                 InetAddressAndPort endpoint = toCassandraInetAddressAndPort(pullFrom);
                 EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
-                MigrationCoordinator.instance.reportEndpointVersion(endpoint, state);
-                MigrationCoordinator.instance.awaitSchemaRequests(TimeUnit.SECONDS.toMillis(10));
+                Gossiper.instance.doOnChangeNotifications(endpoint, ApplicationState.SCHEMA, state.getApplicationState(ApplicationState.SCHEMA));
+                Schema.instance.waitUntilReady(Duration.ofSeconds(10));
             }).accept(pullFrom);
         }
     }
@@ -258,7 +258,7 @@ public class GossipHelper
                 List<Token> tokens = Collections.singletonList(partitioner.getTokenFactory().fromString(tokenString));
                 try
                 {
-                    Collection<InetAddressAndPort> collisions = StorageService.instance.prepareForBootstrap(waitForSchema.toMillis());
+                    Collection<InetAddressAndPort> collisions = StorageService.instance.prepareForBootstrap(waitForSchema.toMillis(), 0);
                     assert collisions.size() == 0 : String.format("Didn't expect any replacements but got %s", collisions);
                     boolean isBootstrapSuccessful = StorageService.instance.bootstrap(tokens, waitForBootstrap.toMillis());
                     assert isBootstrapSuccessful : "Bootstrap did not complete successfully";

test/distributed/org/apache/cassandra/distributed/impl/Instance.java

@@ -35,7 +35,6 @@ import java.util.UUID;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Stream;
 import javax.management.ListenerNotFoundException;
@@ -105,7 +104,6 @@ import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.NoPayload;
 import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.ActiveRepairService;
@@ -633,7 +631,6 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                 StorageService.instance.registerDaemon(CassandraDaemon.getInstanceForTesting());
                 if (config.has(GOSSIP))
                 {
-                    MigrationManager.setUptimeFn(() -> TimeUnit.NANOSECONDS.toMillis(nanoTime() - startedAt));
                     try
                     {
                         StorageService.instance.initServer();
@@ -652,6 +649,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                 }
                 else
                 {
+                    Schema.instance.startSync();
                     Stream peers = cluster.stream().filter(instance -> ((IInstance) instance).isValid());
                     SystemKeyspace.setLocalHostId(config.hostId());
                     if (config.has(BLANK_GOSSIP))
@@ -670,8 +668,6 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                 // see org.apache.cassandra.service.CassandraDaemon.setup
                 StorageService.instance.populateTokenMetadata();
 
-                SystemKeyspace.finishStartup();
-
                 CassandraDaemon.getInstanceForTesting().completeSetup();
 
                 if (config.has(NATIVE_PROTOCOL))

test/distributed/org/apache/cassandra/distributed/test/metric/TableMetricTest.java

@@ -121,7 +121,7 @@ public class TableMetricTest extends TestBaseImpl
         SYSTEM_TABLES = cluster.get(1).callOnInstance(() -> {
             Map<String, Collection<String>> map = new HashMap<>();
             Arrays.asList(SystemKeyspace.metadata(), AuthKeyspace.metadata(), SystemDistributedKeyspace.metadata(),
-                          Schema.getSystemKeyspaceMetadata(), TraceKeyspace.metadata())
+                          SchemaKeyspace.metadata(), TraceKeyspace.metadata())
                   .forEach(meta -> {
                       Set<String> tables = meta.tables.stream().map(t -> t.name).collect(Collectors.toSet());
                       map.put(meta.name, tables);

test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java

@@ -18,13 +18,17 @@
 
 package org.apache.cassandra.distributed.test.ring;
 
+import java.lang.management.ManagementFactory;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.ICluster;
@@ -44,6 +48,27 @@ import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 
 public class BootstrapTest extends TestBaseImpl
 {
+    private long savedMigrationDelay;
+
+    @Before
+    public void beforeTest()
+    {
+        // MigrationCoordinator schedules schema pull requests immediatelly when the node is just starting up, otherwise
+        // the first pull request is sent in 60 seconds. Whether we are starting up or not is detected by examining
+        // the node up-time and if it is lower than MIGRATION_DELAY, we consider the server is starting up.
+        // When we are running multiple test cases in the class, where each starts a node but in the same JVM, the
+        // up-time will be more or less relevant only for the first test. In order to enforce the startup-like behaviour
+        // for each test case, the MIGRATION_DELAY time is adjusted accordingly
+        savedMigrationDelay = CassandraRelevantProperties.MIGRATION_DELAY.getLong();
+        CassandraRelevantProperties.MIGRATION_DELAY.setLong(ManagementFactory.getRuntimeMXBean().getUptime() + savedMigrationDelay);
+    }
+
+    @After
+    public void afterTest()
+    {
+        CassandraRelevantProperties.MIGRATION_DELAY.setLong(savedMigrationDelay);
+    }
+
     @Test
     public void bootstrapTest() throws Throwable
     {

test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.FBUtilities;
@@ -101,11 +102,11 @@ public class BatchStatementBench
     @Setup
     public void setup() throws Throwable
     {
-        Schema.instance.load(KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1)));
+        SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1)), false);
         KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
         TableMetadata metadata = CreateTableStatement.parse(String.format("CREATE TABLE %s (id int, ck int, v int, primary key (id, ck))", table), keyspace).build();
 
-        Schema.instance.load(ksm.withSwapped(ksm.tables.with(metadata)));
+        SchemaTestUtil.addOrUpdateKeyspace(ksm.withSwapped(ksm.tables.with(metadata)), false);
 
         List<ModificationStatement> modifications = new ArrayList<>(batchSize);
         List<List<ByteBuffer>> parameters = new ArrayList<>(batchSize);

test/microbench/org/apache/cassandra/test/microbench/MutationBench.java

@@ -29,6 +29,7 @@ import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.io.util.DataInputBuffer;
@@ -87,7 +88,7 @@ public class MutationBench
     @Setup
     public void setup() throws IOException
     {
-        Schema.instance.load(KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1)));
+        SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1)), false);
         KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
         TableMetadata metadata =
             CreateTableStatement.parse("CREATE TABLE userpics " +
@@ -97,7 +98,7 @@ public class MutationBench
                                        "PRIMARY KEY(userid, picid))", keyspace)
                                 .build();
 
-        Schema.instance.load(ksm.withSwapped(ksm.tables.with(metadata)));
+        SchemaTestUtil.addOrUpdateKeyspace(ksm.withSwapped(ksm.tables.with(metadata)), false);
 
         mutation = (Mutation)UpdateBuilder.create(metadata, 1L).newRow(1L).add("commentid", 32L).makeMutation();
         buffer = ByteBuffer.allocate(mutation.serializedSize(MessagingService.current_version));

test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceSyncSchemaForBootstrap.java

@@ -18,7 +18,9 @@
 
 package org.apache.cassandra.simulator.cluster;
 
-import org.apache.cassandra.schema.MigrationCoordinator;
+import java.time.Duration;
+
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.simulator.systems.SimulatedActionTask;
 
 import static org.apache.cassandra.simulator.Action.Modifier.DISPLAY_ORIGIN;
@@ -29,6 +31,6 @@ class OnInstanceSyncSchemaForBootstrap extends SimulatedActionTask
     public OnInstanceSyncSchemaForBootstrap(ClusterActions actions, int node)
     {
         super("Sync Schema on " + node, RELIABLE_NO_TIMEOUTS.with(DISPLAY_ORIGIN), RELIABLE_NO_TIMEOUTS, actions, actions.cluster.get(node),
-              () -> MigrationCoordinator.instance.awaitSchemaRequests(Long.MAX_VALUE));
+              () -> Schema.instance.waitUntilReady(Duration.ofMinutes(10)));
     }
 }

test/unit/org/apache/cassandra/SchemaLoader.java

@@ -247,7 +247,7 @@ public class SchemaLoader
         // if you're messing with low-level sstable stuff, it can be useful to inject the schema directly
         // Schema.instance.load(schemaDefinition());
         for (KeyspaceMetadata ksm : schema)
-            MigrationManager.announceNewKeyspace(ksm, false);
+            SchemaTestUtil.announceNewKeyspace(ksm);
 
         if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
             useCompression(schema, compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH));
@@ -255,7 +255,7 @@ public class SchemaLoader
 
     public static void createKeyspace(String name, KeyspaceParams params)
     {
-        MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of()), true);
+        SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of()));
     }
 
     public static void createKeyspace(String name, KeyspaceParams params, TableMetadata.Builder... builders)
@@ -264,17 +264,17 @@ public class SchemaLoader
         for (TableMetadata.Builder builder : builders)
             tables.add(builder.build());
 
-        MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables.build()), true);
+        SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables.build()));
     }
 
     public static void createKeyspace(String name, KeyspaceParams params, TableMetadata... tables)
     {
-        MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of(tables)), true);
+        SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of(tables)));
     }
 
     public static void createKeyspace(String name, KeyspaceParams params, Tables tables, Types types)
     {
-        MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables, Views.none(), types, Functions.none()), true);
+        SchemaTestUtil.announceNewKeyspace(KeyspaceMetadata.create(name, params, tables, Views.none(), types, Functions.none()));
     }
 
     public static void setupAuth(IRoleManager roleManager, IAuthenticator authenticator, IAuthorizer authorizer, INetworkAuthorizer networkAuthorizer)
@@ -283,7 +283,7 @@ public class SchemaLoader
         DatabaseDescriptor.setAuthenticator(authenticator);
         DatabaseDescriptor.setAuthorizer(authorizer);
         DatabaseDescriptor.setNetworkAuthorizer(networkAuthorizer);
-        MigrationManager.announceNewKeyspace(AuthKeyspace.metadata(), true);
+        SchemaTestUtil.announceNewKeyspace(AuthKeyspace.metadata());
         DatabaseDescriptor.getRoleManager().setup();
         DatabaseDescriptor.getAuthenticator().setup();
         DatabaseDescriptor.getAuthorizer().setup();
@@ -335,7 +335,7 @@ public class SchemaLoader
     {
         for (KeyspaceMetadata ksm : schema)
             for (TableMetadata cfm : ksm.tablesAndViews())
-                MigrationManager.announceTableUpdate(cfm.unbuild().compression(compressionParams.copy()).build(), true);
+                SchemaTestUtil.announceTableUpdate(cfm.unbuild().compression(compressionParams.copy()).build());
     }
 
     public static TableMetadata.Builder counterCFMD(String ksName, String cfName)

test/unit/org/apache/cassandra/cql3/CQLTester.java

@@ -479,7 +479,7 @@ public abstract class CQLTester
         };
 
         DatabaseDescriptor.setRoleManager(roleManager);
-        MigrationManager.announceNewKeyspace(AuthKeyspace.metadata(), true);
+        SchemaTestUtil.addOrUpdateKeyspace(AuthKeyspace.metadata(), true);
         DatabaseDescriptor.getRoleManager().setup();
         DatabaseDescriptor.getAuthenticator().setup();
         DatabaseDescriptor.getAuthorizer().setup();
@@ -514,7 +514,6 @@ public abstract class CQLTester
 
     private static void startServices()
     {
-        SystemKeyspace.finishStartup();
         VirtualKeyspaceRegistry.instance.register(VirtualSchemaKeyspace.instance);
         StorageService.instance.initServer();
         SchemaLoader.startGossiper();

test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java

@@ -30,12 +30,15 @@ import org.junit.Test;
 
 import org.apache.cassandra.cache.KeyCacheKey;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.metrics.CacheMetrics;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.schema.CachingParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaKeyspace;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.StorageService;
@@ -309,7 +312,7 @@ public class KeyCacheCqlTest extends CQLTester
         }
 
         dropTable("DROP TABLE %s");
-        Schema.instance.updateVersion();
+        assert Schema.instance.isSameVersion(SchemaKeyspace.calculateSchemaDigest());
 
         //Test loading for a dropped 2i/table
         CacheService.instance.keyCache.clear();

test/unit/org/apache/cassandra/cql3/ViewSchemaTest.java

@@ -848,8 +848,9 @@ public class ViewSchemaTest extends ViewAbstractTest
 
         String view = createView(createView);
 
-        ColumnFamilyStore mv = Keyspace.open(keyspace()).getColumnFamilyStore(view);
-        assertTrue(SchemaCQLHelper.getTableMetadataAsCQL(mv.metadata())
+        Keyspace keyspace = Keyspace.open(keyspace());
+        ColumnFamilyStore mv = keyspace.getColumnFamilyStore(view);
+        assertTrue(SchemaCQLHelper.getTableMetadataAsCQL(mv.metadata(), keyspace.getMetadata())
                                   .startsWith(String.format(viewSnapshotSchema,
                                                             keyspace(),
                                                             view,

test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java

@@ -33,7 +33,6 @@ import org.junit.Test;
 
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.SchemaCQLHelper;
 import org.apache.cassandra.db.marshal.TupleType;
 import org.apache.cassandra.utils.AbstractTypeGenerators.TypeSupport;
 import org.quicktheories.core.Gen;

test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java

@@ -38,6 +38,7 @@ import org.apache.cassandra.db.marshal.CollectionType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
@@ -860,7 +861,7 @@ public class UFTest extends CQLTester
                                                             "java",
                                                             f.body(),
                                                             new InvalidRequestException("foo bar is broken"));
-        Schema.instance.load(ksm.withSwapped(ksm.functions.without(f.name(), f.argTypes()).with(broken)));
+        SchemaTestUtil.addOrUpdateKeyspace(ksm.withSwapped(ksm.functions.without(f.name(), f.argTypes()).with(broken)), false);
 
         assertInvalidThrowMessage("foo bar is broken", InvalidRequestException.class,
                                   "SELECT key, " + fName + "(dval) FROM %s");

test/unit/org/apache/cassandra/cql3/validation/operations/CompactStorageTest.java

@@ -4638,7 +4638,7 @@ public class CompactStorageTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_compact.test_table_compact (\n" +
                           "    pk1 varint,\n" +
                           "    pk2 ascii,\n" +
@@ -4674,7 +4674,7 @@ public class CompactStorageTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_counter.test_table_counter (\n" +
                           "    pk1 varint,\n" +
                           "    pk2 ascii,\n" +
@@ -4699,7 +4699,7 @@ public class CompactStorageTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
 
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
                         "    pk1 varint,\n" +
                         "    reg1 int,\n" +
@@ -4721,7 +4721,7 @@ public class CompactStorageTest extends CQLTester
                                        " WITH COMPACT STORAGE");
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
-        assertTrue(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata()).contains(
+        assertTrue(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata()).contains(
         "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
         "    pk1 varint,\n" +
         "    reg1 int,\n" +
@@ -4743,7 +4743,7 @@ public class CompactStorageTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
 
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
                           "    pk1 varint,\n" +
                           "    reg1 counter,\n" +
@@ -4766,7 +4766,7 @@ public class CompactStorageTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
 
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
                           "    pk1 varint,\n" +
                           "    ck1 int,\n" +
@@ -4789,7 +4789,7 @@ public class CompactStorageTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
 
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS " + keyspace() + "." + tableName + " (\n" +
                           "    pk1 varint,\n" +
                           "    ck1 int,\n" +

test/unit/org/apache/cassandra/cql3/validation/operations/CompactTableTest.java

@@ -27,6 +27,7 @@ import org.apache.cassandra.cql3.QueryHandler;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaChangeListener;
+import org.apache.cassandra.schema.TableMetadata;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -71,7 +72,8 @@ public class CompactTableTest extends CQLTester
         // Verify that schema change listeners are told statements are affected with DROP COMPACT STORAGE.
         SchemaChangeListener listener = new SchemaChangeListener()
         {
-            public void onAlterTable(String keyspace, String table, boolean affectsStatements)
+            @Override
+            public void onAlterTable(TableMetadata before, TableMetadata after, boolean affectsStatements)
             {
                 assertTrue(affectsStatements);
             }

test/unit/org/apache/cassandra/db/CounterCacheTest.java

@@ -22,6 +22,8 @@ import java.util.concurrent.ExecutionException;
 
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.KeyspaceParams;
@@ -191,7 +193,8 @@ public class CounterCacheTest
         CacheService.instance.invalidateCounterCache();
         assertEquals(0, CacheService.instance.counterCache.size());
 
-        Keyspace ks = Schema.instance.maybeRemoveKeyspaceInstance(KEYSPACE1);
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE1);
+        SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE1, true);
 
         try
         {
@@ -201,7 +204,7 @@ public class CounterCacheTest
         }
         finally
         {
-            Schema.instance.maybeAddKeyspaceInstance(ks.getName(), () -> ks);
+            SchemaTestUtil.addOrUpdateKeyspace(ksm, true);
         }
     }
 

test/unit/org/apache/cassandra/db/KeyspaceTest.java

@@ -21,12 +21,9 @@ package org.apache.cassandra.db;
 import java.nio.ByteBuffer;
 import java.util.*;
 
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.schema.Schema;
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
-import org.mockito.Mockito;
 
 import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.CQLTester;
@@ -40,16 +37,10 @@ import org.apache.cassandra.db.filter.*;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.ClearableHistogram;
-import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.junit.Assert.*;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
 
 public class KeyspaceTest extends CQLTester
 {

test/unit/org/apache/cassandra/db/RangeTombstoneTest.java

@@ -45,8 +45,8 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -320,7 +320,7 @@ public class RangeTombstoneTest
     {
         Keyspace ks = Keyspace.open(KSNAME);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build());
 
         String key = "7810";
 
@@ -343,7 +343,7 @@ public class RangeTombstoneTest
     {
         Keyspace ks = Keyspace.open(KSNAME);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build());
 
         String key = "7808_1";
         UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
@@ -363,7 +363,7 @@ public class RangeTombstoneTest
     {
         Keyspace ks = Keyspace.open(KSNAME);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(2).build());
 
         String key = "7808_2";
         UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0);
@@ -489,7 +489,7 @@ public class RangeTombstoneTest
                 current.unbuild()
                        .indexes(current.indexes.with(indexDef))
                        .build();
-            MigrationManager.announceTableUpdate(updated, true);
+            SchemaTestUtil.announceTableUpdate(updated);
         }
 
         Future<?> rebuild = cfs.indexManager.addIndex(indexDef, false);
@@ -595,7 +595,7 @@ public class RangeTombstoneTest
                 current.unbuild()
                        .indexes(current.indexes.with(indexDef))
                        .build();
-            MigrationManager.announceTableUpdate(updated, true);
+            SchemaTestUtil.announceTableUpdate(updated);
         }
 
         Future<?> rebuild = cfs.indexManager.addIndex(indexDef, false);

test/unit/org/apache/cassandra/db/ReadCommandTest.java

@@ -72,6 +72,7 @@ import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.service.ActiveRepairService;
@@ -902,10 +903,7 @@ public class ReadCommandTest
     {
         TableParams newParams = cfs.metadata().params.unbuild().gcGraceSeconds(gcGrace).build();
         KeyspaceMetadata keyspaceMetadata = Schema.instance.getKeyspaceMetadata(cfs.metadata().keyspace);
-        Schema.instance.load(
-        keyspaceMetadata.withSwapped(
-        keyspaceMetadata.tables.withSwapped(
-        cfs.metadata().withSwapped(newParams))));
+        SchemaTestUtil.addOrUpdateKeyspace(keyspaceMetadata.withSwapped(keyspaceMetadata.tables.withSwapped(cfs.metadata().withSwapped(newParams))), true);
     }
 
     private long getAndResetOverreadCount(ColumnFamilyStore cfs)

test/unit/org/apache/cassandra/db/RowCacheTest.java

@@ -33,6 +33,7 @@ import org.apache.cassandra.Util;
 import org.apache.cassandra.cache.RowCacheKey;
 import org.apache.cassandra.db.marshal.ValueAccessors;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.rows.*;
@@ -48,6 +49,7 @@ import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.metrics.ClearableHistogram;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -367,7 +369,9 @@ public class RowCacheTest
         CacheService.instance.setRowCacheCapacityInMB(1);
         rowCacheLoad(100, 50, 0);
         CacheService.instance.rowCache.submitWrite(Integer.MAX_VALUE).get();
-        Keyspace instance = Schema.instance.maybeRemoveKeyspaceInstance(KEYSPACE_CACHED);
+
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE_CACHED);
+        SchemaTestUtil.dropKeyspaceIfExist(KEYSPACE_CACHED, true);
         try
         {
             CacheService.instance.rowCache.size();
@@ -378,7 +382,7 @@ public class RowCacheTest
         }
         finally
         {
-            Schema.instance.maybeAddKeyspaceInstance(instance.getName(), () -> instance);
+            SchemaTestUtil.addOrUpdateKeyspace(ksm, true);
         }
     }
 

test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java

@@ -167,7 +167,7 @@ public class SchemaCQLHelperTest extends CQLTester
                           "    reg2 varint,\n" +
                           "    st1 varint static,\n" +
                           "    PRIMARY KEY (pk1, ck1)\n) WITH ID =";
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
 
         assertThat(actual,
                    allOf(startsWith(expected),
@@ -208,7 +208,7 @@ public class SchemaCQLHelperTest extends CQLTester
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
         // when re-adding, column is present as both column and as dropped column record.
-        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata());
+        String actual = SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata());
         String expected = "CREATE TABLE IF NOT EXISTS cql_test_keyspace_readded_columns.test_table_readded_columns (\n" +
                           "    pk1 varint,\n" +
                           "    ck1 varint,\n" +
@@ -247,7 +247,7 @@ public class SchemaCQLHelperTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
-        assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata()),
+        assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata()),
                    startsWith(
                    "CREATE TABLE IF NOT EXISTS cql_test_keyspace_create_table.test_table_create_table (\n" +
                    "    pk1 varint,\n" +
@@ -294,7 +294,7 @@ public class SchemaCQLHelperTest extends CQLTester
 
         ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
 
-        assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata()),
+        assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata()),
                    containsString("CLUSTERING ORDER BY (cl1 ASC)\n" +
                             "    AND additional_write_policy = 'ALWAYS'\n" +
                             "    AND bloom_filter_fp_chance = 1.0\n" +

test/unit/org/apache/cassandra/db/SecondaryIndexTest.java

@@ -40,8 +40,8 @@ import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -482,7 +482,7 @@ public class SecondaryIndexTest
             current.unbuild()
                    .indexes(current.indexes.with(indexDef))
                    .build();
-        MigrationManager.announceTableUpdate(updated, true);
+        SchemaTestUtil.announceTableUpdate(updated);
 
         // wait for the index to be built
         Index index = cfs.indexManager.getIndex(indexDef);

test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java

@@ -25,18 +25,18 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspace;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.FBUtilities;
 
 import static java.lang.String.format;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
@@ -121,7 +121,7 @@ public class SystemKeyspaceTest
         Set<String> snapshottedSystemTables = getSystemSnapshotFiles(SchemaConstants.SYSTEM_KEYSPACE_NAME);
         SystemKeyspace.metadata().tables.forEach(t -> assertTrue(snapshottedSystemTables.contains(t.name)));
         Set<String> snapshottedSchemaTables = getSystemSnapshotFiles(SchemaConstants.SCHEMA_KEYSPACE_NAME);
-        Schema.getSystemKeyspaceMetadata().tables.forEach(t -> assertTrue(snapshottedSchemaTables.contains(t.name)));
+        SchemaKeyspace.metadata().tables.forEach(t -> assertTrue(snapshottedSchemaTables.contains(t.name)));
 
         // clear out the snapshots & set the previous recorded version equal to the latest, we shouldn't
         // see any new snapshots created this time.

test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java

@@ -160,8 +160,8 @@ public abstract class CommitLogTest
                                     custom);
         SchemaLoader.createKeyspace(KEYSPACE2,
                                     KeyspaceParams.simpleTransient(1),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance),
-                                    SchemaLoader.standardCFMD(KEYSPACE1, STANDARD2, 0, AsciiType.instance, BytesType.instance));
+                                    SchemaLoader.standardCFMD(KEYSPACE2, STANDARD1, 0, AsciiType.instance, BytesType.instance),
+                                    SchemaLoader.standardCFMD(KEYSPACE2, STANDARD2, 0, AsciiType.instance, BytesType.instance));
         CompactionManager.instance.disableAutoCompaction();
 
         testKiller = new KillerForTests();

test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java

@@ -42,6 +42,7 @@ import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
@@ -127,7 +128,10 @@ public class CommitLogUpgradeTest
         {
             TableId tableId = TableId.fromString(cfidString);
             if (Schema.instance.getTableMetadata(tableId) == null)
-                Schema.instance.load(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1), Tables.of(metadata.unbuild().id(tableId).build())));
+                SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(KEYSPACE,
+                                                                           KeyspaceParams.simple(1),
+                                                                           Tables.of(metadata.unbuild().id(tableId).build())),
+                                                   true);
         }
 
         Hasher hasher = new Hasher();

test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java

@@ -55,7 +55,7 @@ public class CompactionTaskTest
     public static void setUpClass() throws Exception
     {
         SchemaLoader.prepareServer();
-        cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "coordinatorsessiontest").build();
+        cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", "ks").build();
         SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), cfm);
         cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id);
     }

test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Throwables;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMRules;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
@@ -216,7 +217,7 @@ public class CompactionsBytemanTest extends CQLTester
         }
         catch (RuntimeException t)
         {
-            if (!(t.getCause().getCause() instanceof CompactionInterruptedException))
+            if (!Throwables.isCausedBy(t, CompactionInterruptedException.class::isInstance))
                 throw t;
             //expected
         }

test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java

@@ -68,7 +68,7 @@ import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
@@ -131,7 +131,7 @@ public class CompactionsTest
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
         store.clearUnsafe();
-        MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).build(), true);
+        SchemaTestUtil.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).build());
 
         // disable compaction while flushing
         store.disableAutoCompaction();
@@ -173,7 +173,7 @@ public class CompactionsTest
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
         store.clearUnsafe();
 
-        MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build(), true);
+        SchemaTestUtil.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build());
 
         // disable compaction while flushing
         store.disableAutoCompaction();
@@ -216,7 +216,7 @@ public class CompactionsTest
 
         // now let's enable the magic property
         compactionOptions.put("unchecked_tombstone_compaction", "true");
-        MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build(), true);
+        SchemaTestUtil.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build());
 
         //submit background task again and wait for it to complete
         FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));

test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java

@@ -29,6 +29,7 @@ import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.ColumnFilter;
@@ -40,7 +41,6 @@ import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.tools.SSTableExpiredBlockers;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
@@ -80,7 +80,7 @@ public class TTLExpiryTest
     {
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
         cfs.disableAutoCompaction();
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build());
         String key = "ttl";
         new RowUpdateBuilder(cfs.metadata(), 1L, 1, key)
                     .add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER)
@@ -161,7 +161,7 @@ public class TTLExpiryTest
         cfs.truncateBlocking();
         cfs.disableAutoCompaction();
         // To reproduce #10944, we need our gcBefore to be equal to the locaDeletionTime. A gcGrace of 1 will (almost always) give us that.
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(force10944Bug ? 1 : 0).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(force10944Bug ? 1 : 0).build());
         long timestamp = System.currentTimeMillis();
         String key = "ttl";
         new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
@@ -209,7 +209,7 @@ public class TTLExpiryTest
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
         cfs.truncateBlocking();
         cfs.disableAutoCompaction();
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build());
         long timestamp = System.currentTimeMillis();
         String key = "ttl";
         new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key)
@@ -259,7 +259,7 @@ public class TTLExpiryTest
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
         cfs.truncateBlocking();
         cfs.disableAutoCompaction();
-        MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
+        SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build());
 
         new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), "test")
                 .noRowMarker()

test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java

@@ -67,7 +67,7 @@ import org.apache.cassandra.net.BufferPoolAllocator;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.SharedDefaultFileRegion;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.MigrationManager;
+import org.apache.cassandra.schema.SchemaTestUtil;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory;
@@ -247,7 +247,7 @@ public class EntireSSTableStreamConcurrentComponentMutationTest
 
         // rewrite index summary file with new min/max index interval
         TableMetadata origin = store.metadata();
-        MigrationManager.announceTableUpdate(origin.unbuild().minIndexInterval(1).maxIndexInterval(2).build(), true);
+        SchemaTestUtil.announceTableUpdate(origin.unbuild().minIndexInterval(1).maxIndexInterval(2).build());
 
         try (LifecycleTransaction txn = store.getTracker().tryModify(sstable, OperationType.INDEX_SUMMARY))
         {
@@ -257,7 +257,7 @@ public class EntireSSTableStreamConcurrentComponentMutationTest
         }
 
         // reset min/max index interval
-        MigrationManager.announceTableUpdate(origin, true);
+        SchemaTestUtil.announceTableUpdate(origin);
         return true;
     }