diff --git a/NEWS.txt b/NEWS.txt index b493e48d51..22921153da 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -108,7 +108,6 @@ New features the respective configuration sections and by commenting out `configure_jmx` function call in cassandra-env.sh. Enabling both ways of configuring JMX will result in a node failing to start. - Upgrading --------- [The following is a placeholder, to be revised asap] @@ -119,12 +118,12 @@ Upgrading 2. The Cluster Metadata Service (CMS) has been enabled The first step is just a regular upgrade, there are no changes to external APIs, SSTable formats to consider. Step 2 requires an operator to run a new nodetool subcommand, intializecms, on one node in the cluster. - - > nodetool initializecms - Doing so creates the CMS with the local node as its only member. The initializecms command cannot be executed until + - > nodetool cms initialize + Doing so creates the CMS with the local node as its only member. The initialize command cannot be executed until step 1 is complete and all nodes are running 5.1 as migrating metadata management over to the new TCM system requires all nodes to be in agreement over the current state of the cluster. Essentially this means agreement on - schema and topology. Once the upgrade has started, but before initializecms is run, metadata-changing operations are - not permitted and if attempted from an upgraded node will be rejected. + schema and topology. Once the upgrade has started, but before the initialize command is run, metadata-changing + operations are not permitted and if attempted from an upgraded node will be rejected. Prohibited operations include: - schema changes - node replacement @@ -136,22 +135,22 @@ Upgrading a previous version, though this is planned before release. Any automation which can trigger these operations should be disabled for the cluster prior to starting the upgrade. Should any of the prohibited operations be executed (i.e. on a node that is still running a pre-5.1 version) before - the CMS migration, nodes which are DOWN or which have been upgraded will not process the metadata changes. However, - nodes still UP and running the old version will. This will eventually cause the migration to fail, as the cluster - will not be in agreement. - - > nodetool initializecms + the CMS intialization, nodes which are DOWN or which have been upgraded will not process the metadata changes. + However, nodes still UP and running the old version will. This will eventually cause the migration to fail, as the + cluster will not be in agreement. + - > nodetool cms initialize Got mismatching cluster metadatas from [/x.x.x.x:7000] aborting migration See 'nodetool help' or 'nodetool help '. - If the initializecms command fails, it will indicate which nodes’ current metadata does not agree with the node + If the cms initialize command fails, it will indicate which nodes’ current metadata does not agree with the node where the command was executed. To mitigate this situation, bring any mismatching nodes DOWN and rerun the - initializecms command with the additional —ignore flag. - - nodetool intializecms -ignore x.x.x.x + cms initialize command with the additional —ignore flag. + - nodetool cms intialize -ignore x.x.x.x Once the command has run successfully the ignored nodes can be restarted but any metadata changes that they accepted/and or applied whilst the cluster was in mixed mode will be lost. We plan to improve this before beta, but in the meantime operators should ensure no schema or membership/ownership changes are performed during the upgrade. Although the restrictions on metadata-changing operations will be removed as soon as the initial CMS migration is complete, at that point the CMS will only contain a single member, which is not suitable for real clusters. To - modify the membership of the CMS a second nodetool subcommand, reconfigurecms, should be run. This command allows + modify the membership of the CMS a second nodetool subcommand, cms reconfigure, should be run. This command allows the number of required number of CMS members to be specified for each datacenter. Consensus for metadata operations by the CMS is obtained via Paxos, operating at SERIAL/QUORUM consistency, so the minimum safe size for the CMS is 3 nodes. In multi-DC deployments, the CMS members may be distributed across DCs to ensure resilience in the case of a @@ -173,7 +172,23 @@ Upgrading Deprecation ----------- - `use_deterministic_table_id` is no longer supported and should be removed from cassandra.yaml. Table IDs may still be supplied explicitly on CREATE. - + - IEndpointSnitch has been deprecated as ClusterMetadata is now the source of truth regarding topology information. + The responsibilities of the snitch have been broken out to a handful of new classes: + * o.a.c.locator.Locator provides datacenter and rack info for endpoints. This is not configurable as topology is + always sourced from ClusterMetadata. + * o.a.c.locator.InitialLocationProvider supplies the local datacenter and rack for a new node joining a cluster + for the first time. This will be used exactly once to register the Location of the node in ClusterMetadata and + is configurable using the `initial_location_provider` yaml setting. + * o.a.c.locator.NodeProximity handles sorting and ranking of replica lists. It is configurable via the + `node_proximity` yaml setting. + * o.a.c.locator.NodeAddressConfig exists mainly to support the functionality of ReconnectableSnitchHelper and + Ec2MultiRegionSnitch which dynamically configures the broadcast address based on cloud metadata. Optionally + configurable using the `address_config` yaml setting. + For migration and to allow us to deprecate snitches in a controlled way, use of endpoint_snitch in yaml remains + fully supported and all existing IEndpointSnitch implementations should continue to work seamlessly and no action + is required by operators at upgrade time. The in-tree snitches have been refactored to extract implementations of + the new interfaces so that their functionality can also be used via the new `initial_location_provider`, + `node_proximity` and `address_config` settings. 5.0 === diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index f8d9f354cb..6ffe1fb9b6 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1465,6 +1465,9 @@ slow_query_log_timeout: 500ms # most users should never need to adjust this. # phi_convict_threshold: 8 +# IEndpointSnitch has been deprecated in Cassandra 5.1 +# Configuring a cluster with an IEndpointSnitch implementation using the endpoint_snitch setting remains supported, +# but is superceded by the new settings detailed below. # endpoint_snitch -- Set this to a class that implements # IEndpointSnitch. The snitch has two functions: # @@ -1550,6 +1553,97 @@ slow_query_log_timeout: 500ms # of the snitch, which will be assumed to be on your classpath. endpoint_snitch: SimpleSnitch +# The settings in the following section are intended to supercede the use of endpoint_snitch: +# initial_location_provider +# node_proximity +# addresses_config (optional) +# prefer_local_connections (optional, defaults to false) + +# The initial location provider supplies the datacenter and rack for a new node joining the +# cluster for the first time. This DC/rack is used to register the node with the cluster and +# is then persisted in and propagated by cluster metadata. The initial location provider is +# not used after this initial phase of the node lifecycle is complete. Cassandra provides the +# following implementations: +# +# SimpleLocationProvider: +# Hardcoded to supply a static location of `datacenter1/rack1`. Replicates the behaviour of +# SimpleSnitch. +# +# RackDCFileLocationProvider: +# The rack and datacenter for the local node are defined in cassandra-rackdc.properties, +# and is backwards compatible with the configuration of GossipingPropertyFileSnitch. +# +# TopologyFileLocationProvider: +# The rack and datacenter for the local node are defined in cassandra-topology.properties, +# and is backwards compatible with the configuration of PropertyFileSnitch. Only the location +# info for the local node is relevant now, rack/dc specifications for other endpoints are +# ignored. +# +# AlibabaCloudLocationProvider: +# Obtains datacenter and rack of the node from the metadata service of Alibaba cloud. This +# maps the ECS region to datacenter and the ECS availability zone to the rack. Intended to +# replace use of AlibabaCloudSnitch. +# +# AzureCloudLocationProvider: +# Maps datacenter to `location` and rack to `zone` fields of the `compute` object obtained +# from the Azure metadata service. If the availablity zone is not enabled, it will use the +# failure domain and get its respective value. Intended to replace use of AzureSnitch. +# +# CloudstackLocationProvider: +# Assumes a Cloudstack Zone follows the typical convention country-location-az, using a +# country/location tuple as a datacenter and the availability zone as a rack. Intended to +# replace use of CloudstackSnitch. +# WARNING: This legacy snitch has been deprecated and is scheduled to be removed in a future +# version of Cassandra. +# +# Ec2LocationProvider: +# Loads Region and Availability Zone information from the EC2 API. The Region is treated as the +# datacenter and the AvailablilityZone as the rack. Intended to replace use of Ec2Snitch and +# Ec2MultiRegionSnitch. +# +# GoogleCloudLocationProvider: +# Fetches source data from the metadata service of Google Cloud. This provider maps the GCE +# region to datacenter and the availibility zone to the rack. Intended to replace use of +# GoogleCloudSnitch. +#initial_location_provider: SimpleLocationProvider + +# Node proximity controls sorting and ranking of replica lists in order to route requests efficiently. +# Dynamic snitch (not configured here) provides an additional level of sorting on top of whatever is +# specified here of sorting based on recorded latencies between peers. +# +# Out of the box Cassandra provides: +# +# NoOpProximity: +# Performs no reordering of replica lists and assigns every endpoint equal proximity, regardless of +# actual topology. This replicates the behaviour of SimpleSnitch. +# +# NetworkTopologyProximity: +# Ranks proximity of nodes based on datacenter and rack. Nodes in the same rack are considered +# closest, followed by those in the same datacenter. Nodes in different datacenters are +# considered furthest apart. Replicates the behaviour of topology aware snitches such as +# GossipingPropertyFileSnitch, PropertyFileSnitch and the multiple cloud platform snitches. +#node_proximity: NetworkTopologyProximity + +# Address configuration (public and private endpoint addresses) is almost always derived directly +# from this configuration file using broadcast_address. In certain deployments, both the public and +# private addresses may be configured from an external source. For instance, in EC2 a node will have +# discover both of its own addresses at startup by querying the appropriate cloud metadata service. +# In multi-region EC2 deployments, this should be set to Ec2MultiRegionAddressConfig, to replicate the +# behaviour of Ec2MultiRegionSnitch. +#addresses_config: Ec2MultiRegionAddressConfig + +# Controls whether or not to ensure that connections to peers in the same datacenter are established +# using private addresses. Typically, this is for situations like EC2 where a node will have a public +# address and a private address. We may initially connect on the public, then discover the private, and +# reconnect on the private. +# In Ec2MultiRegionSnitch (now deprecated), this behaviour was hard coded so if migrating from that +# snitch to modern config, set this to true. +# In GossipingPropertyFileSnitch (now deprecated), this behaviour was configured by the `prefer_local` +# property so if migrating from that snitch to modern config, set this accordingly. +# Note that all of the deprecated in-tree snitches can still be used in configuration, so any config +# migration is currently optional. +#prefer_local_connections: false + # controls how often to perform the more expensive part of host score # calculation # Min unit: ms diff --git a/conf/cassandra_latest.yaml b/conf/cassandra_latest.yaml index 109811956d..4f8879a43c 100644 --- a/conf/cassandra_latest.yaml +++ b/conf/cassandra_latest.yaml @@ -1445,90 +1445,94 @@ slow_query_log_timeout: 500ms # most users should never need to adjust this. # phi_convict_threshold: 8 -# endpoint_snitch -- Set this to a class that implements -# IEndpointSnitch. The snitch has two functions: +# The settings in the following section are intended to supercede the use of endpoint_snitch: +# initial_location_provider +# node_proximity +# addresses_config (optional) +# prefer_local_connections (optional, defaults to false) + +# The initial location provider supplies the datacenter and rack for a new node joining the +# cluster for the first time. This DC/rack is used to register the node with the cluster and +# is then persisted in and propagated by cluster metadata. The initial location provider is +# not used after this initial phase of the node lifecycle is complete. Cassandra provides the +# following implementations: # -# - it teaches Cassandra enough about your network topology to route -# requests efficiently -# - it allows Cassandra to spread replicas around your cluster to avoid -# correlated failures. It does this by grouping machines into -# "datacenters" and "racks." Cassandra will do its best not to have -# more than one replica on the same "rack" (which may not actually -# be a physical location) +# SimpleLocationProvider: +# Hardcoded to supply a static location of `datacenter1/rack1`. Replicates the behaviour of +# SimpleSnitch. # -# CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH -# ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss. -# This means that if you start with the default SimpleSnitch, which -# locates every node on "rack1" in "datacenter1", your only options -# if you need to add another datacenter are GossipingPropertyFileSnitch -# (and the older PFS). From there, if you want to migrate to an -# incompatible snitch like Ec2Snitch you can do it by adding new nodes -# under Ec2Snitch (which will locate them in a new "datacenter") and -# decommissioning the old ones. +# RackDCFileLocationProvider: +# The rack and datacenter for the local node are defined in cassandra-rackdc.properties, +# and is backwards compatible with the configuration of GossipingPropertyFileSnitch. # -# Out of the box, Cassandra provides: +# TopologyFileLocationProvider: +# The rack and datacenter for the local node are defined in cassandra-topology.properties, +# and is backwards compatible with the configuration of PropertyFileSnitch. Only the location +# info for the local node is relevant now, rack/dc specifications for other endpoints are +# ignored. # -# SimpleSnitch: -# Treats Strategy order as proximity. This can improve cache -# locality when disabling read repair. Only appropriate for -# single-datacenter deployments. +# AlibabaCloudLocationProvider: +# Obtains datacenter and rack of the node from the metadata service of Alibaba cloud. This +# maps the ECS region to datacenter and the ECS availability zone to the rack. Intended to +# replace use of AlibabaCloudSnitch. # -# GossipingPropertyFileSnitch -# This should be your go-to snitch for production use. The rack -# and datacenter for the local node are defined in -# cassandra-rackdc.properties and propagated to other nodes via -# gossip. If cassandra-topology.properties exists, it is used as a -# fallback, allowing migration from the PropertyFileSnitch. +# AzureCloudLocationProvider: +# Maps datacenter to `location` and rack to `zone` fields of the `compute` object obtained +# from the Azure metadata service. If the availablity zone is not enabled, it will use the +# failure domain and get its respective value. Intended to replace use of AzureSnitch. # -# PropertyFileSnitch: -# Proximity is determined by rack and data center, which are -# explicitly configured in cassandra-topology.properties. +# CloudstackLocationProvider: +# Assumes a Cloudstack Zone follows the typical convention country-location-az, using a +# country/location tuple as a datacenter and the availability zone as a rack. Intended to +# replace use of CloudstackSnitch. +# WARNING: This legacy snitch has been deprecated and is scheduled to be removed in a future +# version of Cassandra. # -# AlibabaCloudSnitch: -# Snitch for getting dc and rack of a node from metadata service of Alibaba cloud. -# This snitch that assumes an ECS region is a DC and an ECS availability_zone is a rack. +# Ec2LocationProvider: +# Loads Region and Availability Zone information from the EC2 API. The Region is treated as the +# datacenter and the AvailablilityZone as the rack. Intended to replace use of Ec2Snitch and +# Ec2MultiRegionSnitch. # -# AzureSnitch: -# Gets datacenter from 'location' and rack from 'zone' fields of 'compute' object -# from instance metadata service. If the availability zone is not enabled, it will use the fault -# domain and get its respective value. +# GoogleCloudLocationProvider: +# Fetches source data from the metadata service of Google Cloud. This provider maps the GCE +# region to datacenter and the availibility zone to the rack. Intended to replace use of +# GoogleCloudSnitch. +initial_location_provider: SimpleLocationProvider + +# Node proximity controls sorting and ranking of replica lists in order to route requests efficiently. +# Dynamic snitch (not configured here) provides an additional level of sorting on top of whatever is +# specified here of sorting based on recorded latencies between peers. # -# CloudstackSnitch: -# A snitch that assumes a Cloudstack Zone follows the typical convention -# country-location-az and uses a country/location tuple as a datacenter -# and the availability zone as a rack. -# WARNING: This snitch is deprecated and it is scheduled to be removed -# in the next major version of Cassandra. +# Out of the box Cassandra provides: # -# Ec2Snitch: -# Appropriate for EC2 deployments in a single Region. Loads Region -# and Availability Zone information from the EC2 API. The Region is -# treated as the datacenter, and the Availability Zone as the rack. -# Only private IPs are used, so this will not work across multiple -# Regions. +# NoOpProximity: +# Performs no reordering of replica lists and assigns every endpoint equal proximity, regardless of +# actual topology. This replicates the behaviour of SimpleSnitch. # -# Ec2MultiRegionSnitch: -# Uses public IPs as broadcast_address to allow cross-region -# connectivity. (Thus, you should set seed addresses to the public -# IP as well.) You will need to open the storage_port or -# ssl_storage_port on the public IP firewall. (For intra-Region -# traffic, Cassandra will switch to the private IP after -# establishing a connection.) -# -# GoogleCloudSnitch: -# Snitch for getting dc and rack of a node from metadata service of Google cloud. -# This snitch that assumes an GCE region is a DC and an GCE availability_zone is a rack. -# -# RackInferringSnitch: -# Proximity is determined by rack and data center, which are -# assumed to correspond to the 3rd and 2nd octet of each node's IP -# address, respectively. Unless this happens to match your -# deployment conventions, this is best used as an example of -# writing a custom Snitch class and is provided in that spirit. -# -# You can use a custom Snitch by setting this to the full class name -# of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: SimpleSnitch +# NetworkTopologyProximity: +# Ranks proximity of nodes based on datacenter and rack. Nodes in the same rack are considered +# closest, followed by those in the same datacenter. Nodes in different datacenters are +# considered furthest apart. Replicates the behaviour of topology aware snitches such as +# GossipingPropertyFileSnitch, PropertyFileSnitch and the multiple cloud platform snitches. +node_proximity: NetworkTopologyProximity + +# Address configuration (public and private endpoint addresses) is almost always derived directly +# from this configuration file using broadcast_address. In certain deployments, both the public and +# private addresses may be configured from an external source. For instance, in EC2 a node will have +# discover both of its own addresses at startup by querying the appropriate cloud metadata service. +# In multi-region EC2 deployments, this should be set to Ec2MultiRegionAddressConfig, to replicate the +# behaviour of Ec2MultiRegionSnitch. +#addresses_config: Ec2MultiRegionAddressConfig + +# Controls whether or not to ensure that connections to peers in the same datacenter are established +# using private addresses. Typically, this is for situations like EC2 where a node will have a public +# address and a private address. We may initially connect on the public, then discover the private, and +# reconnect on the private. +# In Ec2MultiRegionSnitch (now deprecated), this behaviour was hard coded so if migrating from that +# snitch to modern config, set this to true. +# In GossipingPropertyFileSnitch (now deprecated), this behaviour was configured by the `prefer_local` +# property so if migrating from that snitch to modern config, set this accordingly. +#prefer_local_connections: false # controls how often to perform the more expensive part of host score # calculation diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index aaf6167ebb..4d4f6efafd 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -430,6 +430,11 @@ public class Config public DurationSpec.IntMillisecondsBound dynamic_snitch_reset_interval = new DurationSpec.IntMillisecondsBound("10m"); public double dynamic_snitch_badness_threshold = 1.0; + public String node_proximity; + public String initial_location_provider; + public String addresses_config; + public boolean prefer_local_connections = false; + public String failure_detector = "FailureDetector"; public EncryptionOptions.ServerEncryptionOptions server_encryption_options = new EncryptionOptions.ServerEncryptionOptions(); diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 169fc1380f..fe44930ed0 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -32,7 +32,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Comparator; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -90,6 +89,7 @@ import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.fql.FullQueryLoggerOptions; +import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.io.FSWriteError; @@ -105,8 +105,14 @@ import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.EndpointSnitchInfo; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.Locator; +import org.apache.cassandra.locator.LocationInfo; +import org.apache.cassandra.locator.InitialLocationProvider; +import org.apache.cassandra.locator.NodeAddressConfig; +import org.apache.cassandra.locator.ReconnectableSnitchHelper; import org.apache.cassandra.locator.SeedProvider; +import org.apache.cassandra.locator.NodeProximity; +import org.apache.cassandra.locator.SnitchAdapter; import org.apache.cassandra.security.AbstractCryptoProvider; import org.apache.cassandra.security.EncryptionContext; import org.apache.cassandra.security.JREProvider; @@ -114,6 +120,7 @@ import org.apache.cassandra.security.SSLFactory; import org.apache.cassandra.service.CacheService.CacheType; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MBeanWrapper; import org.apache.cassandra.utils.Pair; @@ -185,7 +192,10 @@ public class DatabaseDescriptor static final DurationSpec.LongMillisecondsBound LOWEST_ACCEPTED_TIMEOUT = new DurationSpec.LongMillisecondsBound(10L); private static Supplier newFailureDetector; - private static IEndpointSnitch snitch; + private static NodeProximity nodeProximity; + private static Locator initializationLocator; + private static InitialLocationProvider initialLocationProvider; + private static IEndpointStateChangeSubscriber localAddressReconnector; private static InetAddress listenAddress; // leave null so we can fall through to getLocalHost private static InetAddress broadcastAddress; private static InetAddress rpcAddress; @@ -218,8 +228,6 @@ public class DatabaseDescriptor private static long counterCacheSizeInMiB; private static long indexSummaryCapacityInMiB; - private static String localDC; - private static Comparator localComparator; private static EncryptionContext encryptionContext; private static boolean hasLoggedConfig; @@ -292,6 +300,7 @@ public class DatabaseDescriptor sstableFormats = null; clearMBean("org.apache.cassandra.db:type=DynamicEndpointSnitch"); clearMBean("org.apache.cassandra.db:type=EndpointSnitchInfo"); + clearMBean("org.apache.cassandra.db:type=LocationInfo"); } private static void clearMBean(String name) @@ -352,6 +361,8 @@ public class DatabaseDescriptor applySnitch(); + applyFailureDetector(); + applyEncryptionContext(); } @@ -518,6 +529,8 @@ public class DatabaseDescriptor applySnitch(); + applyFailureDetector(); + applyTokensConfig(); applySeedProvider(); @@ -1484,24 +1497,50 @@ public class DatabaseDescriptor // definitely not safe for tools + clients - implicitly instantiates StorageService public static void applySnitch() { - /* end point snitch */ - if (conf.endpoint_snitch == null) - { - throw new ConfigurationException("Missing endpoint_snitch directive", false); - } - snitch = createEndpointSnitch(conf.dynamic_snitch, conf.endpoint_snitch); - EndpointSnitchInfo.create(); + boolean hasLegacyConfig = conf.endpoint_snitch != null; + boolean hasModernConfig = conf.initial_location_provider != null && conf.node_proximity != null; - localDC = snitch.getLocalDatacenter(); - localComparator = (replica1, replica2) -> { - boolean local1 = localDC.equals(snitch.getDatacenter(replica1)); - boolean local2 = localDC.equals(snitch.getDatacenter(replica2)); - if (local1 && !local2) - return -1; - if (local2 && !local1) - return 1; - return 0; - }; + if (hasLegacyConfig == hasModernConfig) + throw new ConfigurationException("Configuration must specify either node_proximity and " + + "initial_location_provider or endpoint_snitch but not both. "); + + NodeProximity proximity; + NodeAddressConfig addressConfig; + if (hasLegacyConfig) + { + logger.info("Use of endpoint_snitch in configuration is deprecated and should be replaced by " + + "initial_location_provider and node_proximity"); + SnitchAdapter adapter = new SnitchAdapter(createEndpointSnitch(conf.endpoint_snitch)); + proximity = adapter; + initialLocationProvider = adapter; + addressConfig = adapter; + } + else + { + proximity = createProximityImpl(conf.node_proximity); + initialLocationProvider = createInitialLocationProvider(conf.initial_location_provider); + addressConfig = conf.addresses_config != null + ? createAddressConfig(conf.addresses_config) + : NodeAddressConfig.DEFAULT; + } + // this is done here and not in applyAddressConfig as historically, Ec2MultiRegionSnitch is + // responsible for querying the cloud metadata service to get the public IP used for + // broadcast_address and we only want to instantiate the snitch here. + addressConfig.configureAddresses(); + initializationLocator = new Locator(RegistrationStatus.instance, + FBUtilities.getBroadcastAddressAndPort(), + initialLocationProvider); + nodeProximity = conf.dynamic_snitch ? new DynamicEndpointSnitch(proximity) : proximity; + localAddressReconnector = addressConfig.preferLocalConnections() + ? new ReconnectableSnitchHelper(initializationLocator, true) + : new IEndpointStateChangeSubscriber() { /* NO-OP */ }; + + EndpointSnitchInfo.create(); + LocationInfo.create(); + } + + public static void applyFailureDetector() + { newFailureDetector = () -> createFailureDetector(conf.failure_detector); } @@ -1717,12 +1756,36 @@ public class DatabaseDescriptor }); } - public static IEndpointSnitch createEndpointSnitch(boolean dynamic, String snitchClassName) throws ConfigurationException + public static IEndpointSnitch createEndpointSnitch(String snitchClassName) throws ConfigurationException { if (!snitchClassName.contains(".")) snitchClassName = "org.apache.cassandra.locator." + snitchClassName; IEndpointSnitch snitch = FBUtilities.construct(snitchClassName, "snitch"); - return dynamic ? new DynamicEndpointSnitch(snitch) : snitch; + return snitch; + } + + public static NodeProximity createProximityImpl(String className) throws ConfigurationException + { + if (!className.contains(".")) + className = "org.apache.cassandra.locator." + className; + NodeProximity sorter = FBUtilities.construct(className, "node proximity measurement"); + return sorter; + } + + public static InitialLocationProvider createInitialLocationProvider(String className) throws ConfigurationException + { + if (!className.contains(".")) + className = "org.apache.cassandra.locator." + className; + InitialLocationProvider provider = FBUtilities.construct(className, "initial location provider"); + return provider; + } + + public static NodeAddressConfig createAddressConfig(String className) throws ConfigurationException + { + if (!className.contains(".")) + className = "org.apache.cassandra.locator." + className; + NodeAddressConfig config = FBUtilities.construct(className, "node address config"); + return config; } private static IFailureDetector createFailureDetector(String detectorClassName) throws ConfigurationException @@ -2083,14 +2146,49 @@ public class DatabaseDescriptor return old; } - public static IEndpointSnitch getEndpointSnitch() + public static IEndpointStateChangeSubscriber getLocalAddressReconnectionHelper() { - return snitch; + return localAddressReconnector; } - public static void setEndpointSnitch(IEndpointSnitch eps) + public static boolean preferLocalConnections() { - snitch = eps; + return conf.prefer_local_connections; + } + + /** + * Return a Locator that can be used from the moment a node begins its initial startup. + * It is not initialized with a ClusterMetadata instance (as there may not be one yet), so + * instead it always performs lookups using the current metadata. This means it is possible + * for the value of a lookup for the same endpoint to change between calls e.g. if a peer + * were to register or change broadcast address between the two calls being made. + * @return Locator + */ + public static Locator getLocator() + { + if (initializationLocator == null && isClientInitialized()) + return Locator.forClients(); + return initializationLocator; + } + + /** + * Used to provide the location (dc/rack) of the local node during its initial startup + * for the purpose of registering it with ClusterMetadata. + * See: org.apache.cassandra.locator.Locator + */ + public static InitialLocationProvider getInitialLocationProvider() + { + return initialLocationProvider; + } + + public static NodeProximity getNodeProximity() + { + return nodeProximity; + } + + public static void setNodeProximity(NodeProximity proximity) + { + nodeProximity = proximity; } public static IFailureDetector newFailureDetector() @@ -3708,7 +3806,7 @@ public class DatabaseDescriptor public static boolean isDynamicEndpointSnitch() { // not using config.dynamic_snitch because snitch can be changed via JMX - return snitch instanceof DynamicEndpointSnitch; + return nodeProximity instanceof DynamicEndpointSnitch; } public static Config.BatchlogEndpointStrategy getBatchlogEndpointStrategy() @@ -4038,18 +4136,7 @@ public class DatabaseDescriptor public static String getLocalDataCenter() { - return localDC; - } - - @VisibleForTesting - public static void setLocalDataCenter(String value) - { - localDC = value; - } - - public static Comparator getLocalComparator() - { - return localComparator; + return initializationLocator == null ? null : initializationLocator.local().datacenter; } public static Config.InternodeCompression internodeCompression() diff --git a/src/java/org/apache/cassandra/config/EncryptionOptions.java b/src/java/org/apache/cassandra/config/EncryptionOptions.java index 8d922aa96e..1cd3b7addd 100644 --- a/src/java/org/apache/cassandra/config/EncryptionOptions.java +++ b/src/java/org/apache/cassandra/config/EncryptionOptions.java @@ -32,10 +32,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.security.DisableSslContextFactory; import org.apache.cassandra.security.ISslContextFactory; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; @@ -786,7 +787,15 @@ public class EncryptionOptions public boolean shouldEncrypt(InetAddressAndPort endpoint) { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + // When a node is started for the very first time, it has no way to determine whether the seed nodes + // it makes its initial connections to are in a local or remote datacenter and/or rack. When the node is + // in this specific state, Locator will return the constant Location.UNKNOWN for any lookup of a peer's + // location. This is intended to ensure that _all_ peers are treated as remote during this initial phase and + // that the most strict encryption settings allowable by the internode_encryption settings are applied. + // Any connections established during this phase, of which there should be few, will be dropped and + // re-established as soon as the node initialises its local ClusterMetadata and so is able to get accurate + // topology information for peers. + Locator locator = DatabaseDescriptor.getLocator(); switch (internode_encryption) { case none: @@ -794,13 +803,14 @@ public class EncryptionOptions case all: break; case dc: - if (snitch.getDatacenter(endpoint).equals(snitch.getLocalDatacenter())) + if (locator.location(endpoint).datacenter.equals(locator.local().datacenter)) return false; break; case rack: // for rack then check if the DC's are the same. - if (snitch.getRack(endpoint).equals(snitch.getLocalRack()) - && snitch.getDatacenter(endpoint).equals(snitch.getLocalDatacenter())) + Location remote = locator.location(endpoint); + Location local = locator.local(); + if (remote.rack.equals(local.rack) && remote.datacenter.equals(local.datacenter)) return false; break; } diff --git a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java index f32ac48c9c..cd9ffb11fd 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java @@ -41,6 +41,8 @@ import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.RequestValidationException; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.NodeProximity; +import org.apache.cassandra.locator.SnitchAdapter; import org.apache.cassandra.schema.*; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; @@ -674,8 +676,11 @@ public abstract class DescribeStatement extends CQLStatement.Raw implements C List list = new ArrayList(); list.add(DatabaseDescriptor.getClusterName()); list.add(trimIfPresent(DatabaseDescriptor.getPartitionerName(), "org.apache.cassandra.dht.")); - list.add(trimIfPresent(DatabaseDescriptor.getEndpointSnitch().getClass().getName(), - "org.apache.cassandra.locator.")); + NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); + String nodeProximityClassName = proximity instanceof SnitchAdapter ? ((SnitchAdapter) proximity).snitch.getClass().getName() + : proximity.getClass().getName(); + list.add(trimIfPresent(nodeProximityClassName, + "org.apache.cassandra.locator.")); String useKs = state.getRawKeyspace(); if (mustReturnsRangeOwnerships(useKs)) diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java index 73cdeb5c5c..aa83742ac8 100644 --- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java +++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java @@ -18,13 +18,14 @@ package org.apache.cassandra.db; import com.carrotsearch.hppc.ObjectIntHashMap; -import org.apache.cassandra.locator.Endpoints; -import org.apache.cassandra.locator.InOurDc; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.InOurDc; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.transport.ProtocolException; import static org.apache.cassandra.locator.Replicas.addToCountPerDc; @@ -121,10 +122,10 @@ public enum ConsistencyLevel } } - public static ObjectIntHashMap eachQuorumForWrite(AbstractReplicationStrategy replicationStrategy, Endpoints pendingWithDown) + public static ObjectIntHashMap eachQuorumForWrite(Locator locator, AbstractReplicationStrategy replicationStrategy, Endpoints pendingWithDown) { ObjectIntHashMap perDc = eachQuorumForRead(replicationStrategy); - addToCountPerDc(perDc, pendingWithDown, 1); + addToCountPerDc(locator, perDc, pendingWithDown, 1); return perDc; } diff --git a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java index c862174625..d0897afdd5 100644 --- a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java @@ -38,7 +38,7 @@ public class CounterMutationVerbHandler extends AbstractMutationVerbHandler tokensAsSet(Collection tokens) { if (tokens.isEmpty()) diff --git a/src/java/org/apache/cassandra/db/view/ViewUtils.java b/src/java/org/apache/cassandra/db/view/ViewUtils.java index b41150b7a5..510939fe6e 100644 --- a/src/java/org/apache/cassandra/db/view/ViewUtils.java +++ b/src/java/org/apache/cassandra/db/view/ViewUtils.java @@ -22,13 +22,13 @@ import java.util.Optional; import java.util.function.Predicate; import com.google.common.collect.Iterables; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; public final class ViewUtils { @@ -60,7 +60,7 @@ public final class ViewUtils */ public static Optional getViewNaturalEndpoint(ClusterMetadata metadata, String keyspace, Token baseToken, Token viewToken) { - String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); + Location local = metadata.locator.local(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); EndpointsForToken naturalBaseReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(baseToken).get(); @@ -73,7 +73,7 @@ public final class ViewUtils // We only select replicas from our own DC // TODO: this is poor encapsulation, leaking implementation details of replication strategy Predicate isLocalDC = r -> !(keyspaceMetadata.replicationStrategy instanceof NetworkTopologyStrategy) - || DatabaseDescriptor.getEndpointSnitch().getDatacenter(r).equals(localDataCenter); + || metadata.locator.location(r.endpoint()).sameDatacenter(local); // We have to remove any endpoint which is shared between the base and the view, as it will select itself // and throw off the counts otherwise. diff --git a/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java b/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java index 0da5870d3d..ce6e023f5f 100644 --- a/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java +++ b/src/java/org/apache/cassandra/db/virtual/InternodeInboundTable.java @@ -32,6 +32,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.InboundMessageHandlers; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tcm.membership.Location; public final class InternodeInboundTable extends AbstractVirtualTable { @@ -112,9 +113,8 @@ public final class InternodeInboundTable extends AbstractVirtualTable private void addRow(SimpleDataSet dataSet, InetAddressAndPort addressAndPort, InboundMessageHandlers handlers) { - String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(addressAndPort); - String rack = DatabaseDescriptor.getEndpointSnitch().getRack(addressAndPort); - dataSet.row(addressAndPort.getAddress(), addressAndPort.getPort(), dc, rack) + Location location = DatabaseDescriptor.getLocator().location(addressAndPort); + dataSet.row(addressAndPort.getAddress(), addressAndPort.getPort(), location.datacenter, location.rack) .column(USING_BYTES, handlers.usingCapacity()) .column(USING_RESERVE_BYTES, handlers.usingEndpointReserveCapacity()) .column(CORRUPT_FRAMES_RECOVERED, handlers.corruptFramesRecovered()) diff --git a/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java b/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java index 687f845687..5f0351ff44 100644 --- a/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java +++ b/src/java/org/apache/cassandra/db/virtual/InternodeOutboundTable.java @@ -34,6 +34,7 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.OutboundConnection; import org.apache.cassandra.net.OutboundConnections; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tcm.membership.Location; public final class InternodeOutboundTable extends AbstractVirtualTable { @@ -112,10 +113,9 @@ public final class InternodeOutboundTable extends AbstractVirtualTable private void addRow(SimpleDataSet dataSet, InetAddressAndPort addressAndPort, OutboundConnections connections) { - String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(addressAndPort); - String rack = DatabaseDescriptor.getEndpointSnitch().getRack(addressAndPort); + Location location = DatabaseDescriptor.getLocator().location(addressAndPort); long pendingBytes = sum(connections, OutboundConnection::pendingBytes); - dataSet.row(addressAndPort.getAddress(), addressAndPort.getPort(), dc, rack) + dataSet.row(addressAndPort.getAddress(), addressAndPort.getPort(), location.datacenter, location.rack) .column(USING_BYTES, pendingBytes) .column(USING_RESERVE_BYTES, connections.usingReserveBytes()) .column(PENDING_COUNT, sum(connections, OutboundConnection::pendingCount)) diff --git a/src/java/org/apache/cassandra/db/virtual/PendingHintsTable.java b/src/java/org/apache/cassandra/db/virtual/PendingHintsTable.java index 9bab1994e1..900d903b08 100644 --- a/src/java/org/apache/cassandra/db/virtual/PendingHintsTable.java +++ b/src/java/org/apache/cassandra/db/virtual/PendingHintsTable.java @@ -36,10 +36,11 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.FailureDetectorMBean; import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.hints.PendingHintsInfo; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.membership.Location; public final class PendingHintsTable extends AbstractVirtualTable { @@ -81,7 +82,7 @@ public final class PendingHintsTable extends AbstractVirtualTable public DataSet data() { List pendingHints = HintsService.instance.getPendingHintsInfo(); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + Locator locator = DatabaseDescriptor.getLocator(); SimpleDataSet result = new SimpleDataSet(metadata()); @@ -91,6 +92,7 @@ public final class PendingHintsTable extends AbstractVirtualTable else simpleStates = Collections.emptyMap(); + Location location = Location.UNKNOWN; for (PendingHintsInfo info : pendingHints) { InetAddressAndPort addressAndPort = StorageService.instance.getEndpointForHostId(info.hostId); @@ -101,10 +103,11 @@ public final class PendingHintsTable extends AbstractVirtualTable String status = "Unknown"; if (addressAndPort != null) { + location = locator.location(addressAndPort); address = addressAndPort.getAddress(); port = addressAndPort.getPort(); - rack = snitch.getRack(addressAndPort); - dc = snitch.getDatacenter(addressAndPort); + rack = location.rack; + dc = location.datacenter; status = simpleStates.getOrDefault(addressAndPort.toString(), status); } result.row(info.hostId) diff --git a/src/java/org/apache/cassandra/dht/BootStrapper.java b/src/java/org/apache/cassandra/dht/BootStrapper.java index 6c306a3ff0..ebf04dddf8 100644 --- a/src/java/org/apache/cassandra/dht/BootStrapper.java +++ b/src/java/org/apache/cassandra/dht/BootStrapper.java @@ -121,7 +121,7 @@ public class BootStrapper extends ProgressEventNotifierSupport RangeStreamer streamer = new RangeStreamer(metadata, StreamOperation.BOOTSTRAP, useStrictConsistency, - DatabaseDescriptor.getEndpointSnitch(), + DatabaseDescriptor.getNodeProximity(), stateStore, true, DatabaseDescriptor.getStreamingConnectionsPerHost(), diff --git a/src/java/org/apache/cassandra/dht/Datacenters.java b/src/java/org/apache/cassandra/dht/Datacenters.java index 2c0328e8d3..ab44c8a2ce 100644 --- a/src/java/org/apache/cassandra/dht/Datacenters.java +++ b/src/java/org/apache/cassandra/dht/Datacenters.java @@ -28,7 +28,7 @@ public class Datacenters { private static class DCHandle { - private static final String thisDc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); + private static final String thisDc = DatabaseDescriptor.getLocator().local().datacenter; } public static String thisDatacenter() diff --git a/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java b/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java index 949cf99f09..185ec19b90 100644 --- a/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java +++ b/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java @@ -33,12 +33,12 @@ import com.google.common.collect.Multimap; import org.apache.cassandra.locator.EndpointsByRange; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.Replica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.locator.Replicas; import org.psjava.algo.graph.flownetwork.FordFulkersonAlgorithm; import org.psjava.algo.graph.flownetwork.MaximumFlowAlgorithm; @@ -82,14 +82,17 @@ public class RangeFetchMapCalculator private final Vertex sourceVertex = OuterVertex.getSourceVertex(); private final Vertex destinationVertex = OuterVertex.getDestinationVertex(); private final Set> trivialRanges; + private final Locator locator; public RangeFetchMapCalculator(EndpointsByRange rangesWithSources, Collection sourceFilters, - String keyspace) + String keyspace, + Locator locator) { this.rangesWithSources = rangesWithSources; this.sourceFilters = Predicates.and(sourceFilters); this.keyspace = keyspace; + this.locator = locator; this.trivialRanges = rangesWithSources.keySet() .stream() .filter(RangeFetchMapCalculator::isTrivial) @@ -374,7 +377,7 @@ public class RangeFetchMapCalculator private boolean isInLocalDC(Replica replica) { - return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica)); + return locator.local().sameDatacenter(locator.location(replica.endpoint())); } /** diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java index 7cf919b91f..7d57b70956 100644 --- a/src/java/org/apache/cassandra/dht/RangeStreamer.java +++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java @@ -48,15 +48,16 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.EndpointsByRange; import org.apache.cassandra.locator.EndpointsByReplica; import org.apache.cassandra.locator.EndpointsForRange; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; import org.apache.cassandra.locator.Replicas; +import org.apache.cassandra.locator.NodeProximity; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.StreamOperation; @@ -93,7 +94,7 @@ public class RangeStreamer private final List sourceFilters = new ArrayList<>(); private final StreamPlan streamPlan; private final boolean useStrictConsistency; - private final IEndpointSnitch snitch; + private final NodeProximity proximity; private final StreamStateStore stateStore; private final MovementMap movements; private final MovementMap strictMovements; @@ -178,18 +179,18 @@ public class RangeStreamer public static class SingleDatacenterFilter implements SourceFilter { private final String sourceDc; - private final IEndpointSnitch snitch; + private final Locator locator; - public SingleDatacenterFilter(IEndpointSnitch snitch, String sourceDc) + public SingleDatacenterFilter(Locator locator, String sourceDc) { this.sourceDc = sourceDc; - this.snitch = snitch; + this.locator = locator; } @Override public boolean apply(Replica replica) { - return snitch.getDatacenter(replica).equals(sourceDc); + return locator.location(replica.endpoint()).datacenter.equals(sourceDc); } @Override @@ -204,19 +205,19 @@ public class RangeStreamer */ public static class ExcludeLocalDatacenterFilter implements SourceFilter { - private final IEndpointSnitch snitch; + private final Locator locator; private final String localDc; - public ExcludeLocalDatacenterFilter(IEndpointSnitch snitch) + public ExcludeLocalDatacenterFilter(Locator locator) { - this.snitch = snitch; - this.localDc = snitch.getLocalDatacenter(); + this.locator = locator; + this.localDc = locator.local().datacenter; } @Override public boolean apply(Replica replica) { - return !snitch.getDatacenter(replica).equals(localDc); + return !locator.location(replica.endpoint()).datacenter.equals(localDc); } @Override @@ -292,21 +293,21 @@ public class RangeStreamer public RangeStreamer(ClusterMetadata metadata, StreamOperation streamOperation, boolean useStrictConsistency, - IEndpointSnitch snitch, + NodeProximity proximity, StreamStateStore stateStore, boolean connectSequentially, int connectionsPerHost, MovementMap movements, MovementMap strictMovements) { - this(metadata, streamOperation, useStrictConsistency, snitch, stateStore, + this(metadata, streamOperation, useStrictConsistency, proximity, stateStore, FailureDetector.instance, connectSequentially, connectionsPerHost, movements, strictMovements); } RangeStreamer(ClusterMetadata metadata, StreamOperation streamOperation, boolean useStrictConsistency, - IEndpointSnitch snitch, + NodeProximity proximity, StreamStateStore stateStore, IFailureDetector failureDetector, boolean connectSequentially, @@ -319,7 +320,7 @@ public class RangeStreamer this.description = streamOperation.getDescription(); this.streamPlan = new StreamPlan(streamOperation, connectionsPerHost, connectSequentially, null, PreviewKind.NONE); this.useStrictConsistency = useStrictConsistency; - this.snitch = snitch; + this.proximity = proximity; this.stateStore = stateStore; this.movements = movements; this.strictMovements = strictMovements; @@ -368,7 +369,7 @@ public class RangeStreamer } boolean useStrictSource = useStrictSourcesForRanges(keyspace.getMetadata().params.replication, strat); - EndpointsByReplica fetchMap = calculateRangesToFetchWithPreferredEndpoints(snitch::sortedByProximity, + EndpointsByReplica fetchMap = calculateRangesToFetchWithPreferredEndpoints(proximity::sortedByProximity, keyspace.getReplicationStrategy(), useStrictConsistency, metadata, @@ -389,7 +390,7 @@ public class RangeStreamer } else { - workMap = getOptimizedWorkMap(fetchMap, sourceFilters, keyspaceName); + workMap = getOptimizedWorkMap(fetchMap, sourceFilters, keyspaceName, metadata.locator); } if (toFetch.put(keyspaceName, workMap) != null) @@ -458,7 +459,7 @@ public class RangeStreamer * consistency. **/ public static EndpointsByReplica - calculateRangesToFetchWithPreferredEndpoints(BiFunction snitchGetSortedListByProximity, + calculateRangesToFetchWithPreferredEndpoints(BiFunction sortByProximity, AbstractReplicationStrategy strat, boolean useStrictConsistency, ClusterMetadata metadata, @@ -473,7 +474,7 @@ public class RangeStreamer logger.debug("To fetch RN: {}", movements.get(params).keySet()); Predicate testSourceFilters = and(sourceFilters); - Function sorted = endpoints -> snitchGetSortedListByProximity.apply(localAddress, endpoints); + Function sorted = endpoints -> sortByProximity.apply(localAddress, endpoints); //This list of replicas is just candidates. With strict consistency it's going to be a narrow list. EndpointsByReplica.Builder rangesToFetchWithPreferredEndpoints = new EndpointsByReplica.Builder(); @@ -599,7 +600,8 @@ public class RangeStreamer */ private static Multimap getOptimizedWorkMap(EndpointsByReplica rangesWithSources, Collection sourceFilters, - String keyspace) + String keyspace, + Locator locator) { //For now we just aren't going to use the optimized range fetch map with transient replication to shrink //the surface area to test and introduce bugs. @@ -613,7 +615,7 @@ public class RangeStreamer } EndpointsByRange unwrappedView = unwrapped.build(); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(unwrappedView, sourceFilters, keyspace); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(unwrappedView, sourceFilters, keyspace, locator); Multimap> rangeFetchMapMap = calculator.getRangeFetchMap(); logger.info("Output from RangeFetchMapCalculator for keyspace {}", keyspace); validateRangeFetchMap(unwrappedView, rangeFetchMapMap, keyspace); diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocator.java b/src/java/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocator.java index d46c043ed5..0d9eec598a 100644 --- a/src/java/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocator.java +++ b/src/java/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocator.java @@ -23,7 +23,6 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeSet; @@ -32,13 +31,17 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.SimpleSnitch; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.RegistrationStatus; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.OutputHandler; +import static org.apache.cassandra.locator.SimpleLocationProvider.LOCATION; + public class OfflineTokenAllocator { public static List allocate(int rf, int numTokens, int[] nodesPerRack, OutputHandler logger, IPartitioner partitioner) @@ -54,6 +57,9 @@ public class OfflineTokenAllocator Preconditions.checkArgument(nodes >= rf, "not enough nodes %s for rf %s in %s", Arrays.stream(nodesPerRack).sum(), rf, Arrays.toString(nodesPerRack)); + DatabaseDescriptor.setPartitionerUnsafe(partitioner); + // Set RegistrationStatus to REGISTERED so that Locator works exclusively from ClusterMetadata + RegistrationStatus.instance.onRegistration(); List fakeNodes = new ArrayList<>(nodes); MultinodeAllocator allocator = new MultinodeAllocator(rf, numTokens, logger, partitioner); @@ -114,27 +120,21 @@ public class OfflineTokenAllocator private static class MultinodeAllocator { - private final FakeSnitch fakeSnitch; private final TokenAllocation allocation; private final Map lastCheckPoint = Maps.newHashMap(); private final OutputHandler logger; private MultinodeAllocator(int rf, int numTokens, OutputHandler logger, IPartitioner partitioner) { - this.fakeSnitch = new FakeSnitch(); - this.allocation = TokenAllocation.create(fakeSnitch, new ClusterMetadata(partitioner), rf, numTokens); + this.allocation = TokenAllocation.create(LOCATION.datacenter, new ClusterMetadata(partitioner), rf, numTokens); this.logger = logger; } private FakeNode allocateTokensForNode(int nodeId, Integer rackId) { - // Update snitch and token metadata info InetAddressAndPort fakeNodeAddressAndPort = getLoopbackAddressWithPort(nodeId); - fakeSnitch.nodeByRack.put(fakeNodeAddressAndPort, rackId); - // todo: - //fakeMetadata.updateTopology(fakeNodeAddressAndPort); - // Allocate tokens + allocation.addNodeToMetadata(fakeNodeAddressAndPort, new Location(LOCATION.datacenter, rackId.toString())); Collection tokens = allocation.allocate(fakeNodeAddressAndPort); // Validate ownership stats @@ -145,7 +145,7 @@ public class OfflineTokenAllocator private void validateAllocation(int nodeId, int rackId) { - SummaryStatistics newOwnership = allocation.getAllocationRingOwnership(SimpleSnitch.DATA_CENTER_NAME, Integer.toString(rackId)); + SummaryStatistics newOwnership = allocation.getAllocationRingOwnership(LOCATION.datacenter, Integer.toString(rackId)); SummaryStatistics oldOwnership = lastCheckPoint.put(rackId, newOwnership); if (oldOwnership != null) logger.debug(String.format("Replicated node load in rack=%d before allocating node %d: %s.", rackId, nodeId, @@ -164,17 +164,6 @@ public class OfflineTokenAllocator } } - private static class FakeSnitch extends SimpleSnitch - { - final Map nodeByRack = new HashMap<>(); - - @Override - public String getRack(InetAddressAndPort endpoint) - { - return Integer.toString(nodeByRack.get(endpoint)); - } - } - private static InetAddressAndPort getLoopbackAddressWithPort(int port) { try diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java index bad7785ba0..a946cd20ef 100644 --- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java +++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java @@ -24,7 +24,9 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import java.util.function.Supplier; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; @@ -34,23 +36,25 @@ import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractReplicationStrategy; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.membership.NodeVersion; public class TokenAllocation { public static final double WARN_STDEV_GROWTH = 0.05; private static final Logger logger = LoggerFactory.getLogger(TokenAllocation.class); - final ClusterMetadata metadata; + ClusterMetadata metadata; final AbstractReplicationStrategy replicationStrategy; final int numTokens; final Map> strategyByRackDc = new HashMap<>(); @@ -75,14 +79,14 @@ public class TokenAllocation final InetAddressAndPort endpoint, int numTokens) { - return create(DatabaseDescriptor.getEndpointSnitch(), metadata, replicas, numTokens).allocate(endpoint); + return create(metadata.locator.local().datacenter, metadata, replicas, numTokens).allocate(endpoint); } - static TokenAllocation create(IEndpointSnitch snitch, ClusterMetadata metadata, int replicas, int numTokens) + static TokenAllocation create(String localDatacenter, ClusterMetadata metadata, int replicas, int numTokens) { // We create a fake NTS replication strategy with the specified RF in the local DC HashMap options = new HashMap<>(); - options.put(snitch.getLocalDatacenter(), Integer.toString(replicas)); + options.put(localDatacenter, Integer.toString(replicas)); NetworkTopologyStrategy fakeReplicationStrategy = new NetworkTopologyStrategy(null, options); return new TokenAllocation(metadata, fakeReplicationStrategy, numTokens); @@ -93,6 +97,23 @@ public class TokenAllocation return new TokenAllocation(metadata, rs, numTokens); } + @VisibleForTesting + void updateTokensForNode(NodeId id, Collection tokens) + { + metadata = metadata.transformer() + .proposeToken(id, tokens) + .addToRackAndDC(id) // needed by NetworkTopologyStrategy + .build().metadata; + } + + // For use by OfflineTokenAllocator + void addNodeToMetadata(InetAddressAndPort endpoint, Location location) + { + metadata = metadata.transformer() + .register(new NodeAddresses(endpoint), location, NodeVersion.CURRENT) + .build().metadata; + } + Collection allocate(InetAddressAndPort endpoint) { StrategyAdapter strategy = getOrCreateStrategy(endpoint); @@ -100,6 +121,8 @@ public class TokenAllocation tokens = strategy.adjustForCrossDatacenterClashes(tokens); SummaryStatistics os = strategy.replicatedOwnershipStats(); + NodeId nodeId = metadata.directory.peerId(endpoint); + updateTokensForNode(nodeId, tokens); SummaryStatistics ns = strategy.replicatedOwnershipStats(); logger.info("Selected tokens {}", tokens); @@ -136,7 +159,7 @@ public class TokenAllocation // return true iff the provided endpoint occurs in the same virtual token-ring we are allocating for // i.e. the set of the nodes that share ownership with the node we are allocating // alternatively: return false if the endpoint's ownership is independent of the node we are allocating tokens for - abstract boolean inAllocationRing(InetAddressAndPort other); + abstract boolean inAllocationRing(Locator locator, InetAddressAndPort other); final TokenAllocator createAllocator() { @@ -145,7 +168,7 @@ public class TokenAllocation for (Map.Entry en : metadata.tokenMap.asMap().entrySet()) { InetAddressAndPort endpoint = metadata.directory.endpoint(en.getValue()); - if (inAllocationRing(endpoint)) + if (inAllocationRing(metadata.locator, endpoint)) sortedTokens.put(en.getKey(), endpoint); } return TokenAllocatorFactory.createTokenAllocator(sortedTokens, this, metadata.tokenMap.partitioner()); @@ -161,7 +184,7 @@ public class TokenAllocation { NodeId nodeId = metadata.tokenMap.owner(t); InetAddressAndPort other = metadata.directory.endpoint(nodeId); - if (inAllocationRing(other)) + if (inAllocationRing(metadata.locator, other)) throw new ConfigurationException(String.format("Allocated token %s already assigned to node %s. Is another node also allocating tokens?", t, other)); t = t.nextValidToken(); } @@ -176,7 +199,7 @@ public class TokenAllocation for (Map.Entry en : evaluateReplicatedOwnership().entrySet()) { // Filter only in the same allocation ring - if (inAllocationRing(en.getKey())) + if (inAllocationRing(metadata.locator, en.getKey())) { NodeId nodeId = metadata.directory.peerId(en.getKey()); stat.addValue(en.getValue() / metadata.tokenMap.tokens(nodeId).size()); @@ -220,10 +243,8 @@ public class TokenAllocation private StrategyAdapter getOrCreateStrategy(InetAddressAndPort endpoint) { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - String dc = snitch.getDatacenter(endpoint); - String rack = snitch.getRack(endpoint); - return getOrCreateStrategy(dc, rack); + Location location = metadata.locator.location(endpoint); + return getOrCreateStrategy(location.datacenter, location.rack); } private StrategyAdapter getOrCreateStrategy(String dc, String rack) @@ -236,13 +257,13 @@ public class TokenAllocation if (replicationStrategy instanceof NetworkTopologyStrategy) return createStrategy(metadata, (NetworkTopologyStrategy) replicationStrategy, dc, rack); if (replicationStrategy instanceof SimpleStrategy) - return createStrategy((SimpleStrategy) replicationStrategy); + return createStrategy(metadata, (SimpleStrategy) replicationStrategy); throw new ConfigurationException("Token allocation does not support replication strategy " + replicationStrategy.getClass().getSimpleName()); } - private StrategyAdapter createStrategy(final SimpleStrategy rs) + private StrategyAdapter createStrategy(ClusterMetadata metadata, final SimpleStrategy rs) { - return createStrategy(DatabaseDescriptor.getEndpointSnitch(), null, null, rs.getReplicationFactor().allReplicas, false); + return createStrategy(() -> metadata.locator, null, null, rs.getReplicationFactor().allReplicas, false); } private StrategyAdapter createStrategy(ClusterMetadata metadata, NetworkTopologyStrategy strategy, String dc, String rack) @@ -251,6 +272,7 @@ public class TokenAllocation // if topology hasn't been setup yet for this dc+rack then treat it as a separate unit Multimap datacenterRacks = metadata.directory.datacenterRacks(dc); + Supplier locator = () -> metadata.locator; int racks = datacenterRacks != null && datacenterRacks.containsKey(rack) ? datacenterRacks.asMap().size() : 1; @@ -258,21 +280,21 @@ public class TokenAllocation if (replicas <= 1) { // each node is treated as separate and replicates once - return createStrategy(DatabaseDescriptor.getEndpointSnitch(), dc, null, 1, false); + return createStrategy(locator, dc, null, 1, false); } else if (racks == replicas) { // each node is treated as separate and replicates once, with separate allocation rings for each rack - return createStrategy(DatabaseDescriptor.getEndpointSnitch(), dc, rack, 1, false); + return createStrategy(locator, dc, rack, 1, false); } else if (racks > replicas) { // group by rack - return createStrategy(DatabaseDescriptor.getEndpointSnitch(), dc, null, replicas, true); + return createStrategy(locator, dc, null, replicas, true); } else if (racks == 1) { - return createStrategy(DatabaseDescriptor.getEndpointSnitch(), dc, null, replicas, false); + return createStrategy(locator, dc, null, replicas, false); } throw new ConfigurationException(String.format("Token allocation failed: the number of racks %d in datacenter %s is lower than its replication factor %d.", @@ -281,7 +303,7 @@ public class TokenAllocation // a null dc will always return true for inAllocationRing(..) // a null rack will return true for inAllocationRing(..) for all nodes in the same dc - private StrategyAdapter createStrategy(IEndpointSnitch snitch, String dc, String rack, int replicas, boolean groupByRack) + private StrategyAdapter createStrategy(Supplier locator, String dc, String rack, int replicas, boolean groupByRack) { return new StrategyAdapter() { @@ -294,13 +316,14 @@ public class TokenAllocation @Override public Object getGroup(InetAddressAndPort unit) { - return groupByRack ? snitch.getRack(unit) : unit; + return groupByRack ? locator.get().location(unit).rack : unit; } @Override - public boolean inAllocationRing(InetAddressAndPort other) + public boolean inAllocationRing(Locator locator, InetAddressAndPort other) { - return (dc == null || dc.equals(snitch.getDatacenter(other))) && (rack == null || rack.equals(snitch.getRack(other))); + Location location = locator.location(other); + return (dc == null || dc.equals(location.datacenter)) && (rack == null || rack.equals(location.rack)); } }; } diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index de4d9b811b..5274a57ffb 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -1648,13 +1648,12 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, buildSeedsList(); /* initialize the heartbeat state for this localEndpoint */ maybeInitializeLocalState(generationNbr); + register(DatabaseDescriptor.getLocalAddressReconnectionHelper()); + ClusterMetadata metadata = ClusterMetadata.current(); if (mergeLocalStates && metadata.myNodeId() != null) mergeNodeToGossip(metadata.myNodeId(), metadata); - //notify snitches that Gossiper is about to start - DatabaseDescriptor.getEndpointSnitch().gossiperStarting(); - shutdownAnnounced.set(false); scheduledGossipTask = executor.scheduleWithFixedDelay(new GossipTask(), Gossiper.intervalInMillis, diff --git a/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceConnector.java b/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceConnector.java index 2cea1c7145..75f0cfd247 100644 --- a/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceConnector.java +++ b/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceConnector.java @@ -36,11 +36,11 @@ import org.apache.cassandra.exceptions.ConfigurationException; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; -abstract class AbstractCloudMetadataServiceConnector +public abstract class AbstractCloudMetadataServiceConnector { - static final String METADATA_URL_PROPERTY = "metadata_url"; - static final String METADATA_REQUEST_TIMEOUT_PROPERTY = "metadata_request_timeout"; - static final String DEFAULT_METADATA_REQUEST_TIMEOUT = "30s"; + public static final String METADATA_URL_PROPERTY = "metadata_url"; + public static final String METADATA_REQUEST_TIMEOUT_PROPERTY = "metadata_request_timeout"; + public static final String DEFAULT_METADATA_REQUEST_TIMEOUT = "30s"; protected final String metadataServiceUrl; protected final int requestTimeoutMs; diff --git a/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java b/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java index c8563f9b87..e1d4ae66c0 100644 --- a/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java +++ b/src/java/org/apache/cassandra/locator/AbstractCloudMetadataServiceSnitch.java @@ -23,70 +23,32 @@ import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; - -import static java.lang.String.format; - +/** + * @deprecated See CASSANDRA-19488 + */ +@Deprecated(since = "CEP-21") abstract class AbstractCloudMetadataServiceSnitch extends AbstractNetworkTopologySnitch { static final Logger logger = LoggerFactory.getLogger(AbstractCloudMetadataServiceSnitch.class); - static final String DEFAULT_DC = "UNKNOWN-DC"; - static final String DEFAULT_RACK = "UNKNOWN-RACK"; - - protected final AbstractCloudMetadataServiceConnector connector; - - private final String localRack; - private final String localDc; + protected final CloudMetadataLocationProvider locationProvider; private Map> savedEndpoints; - public AbstractCloudMetadataServiceSnitch(AbstractCloudMetadataServiceConnector connector, Pair dcAndRack) + public AbstractCloudMetadataServiceSnitch(CloudMetadataLocationProvider locationProvider) { - this.connector = connector; - this.localDc = dcAndRack.left; - this.localRack = dcAndRack.right; - - logger.info(format("%s using datacenter: %s, rack: %s, connector: %s, properties: %s", - getClass().getName(), getLocalDatacenter(), getLocalRack(), connector, connector.getProperties())); + this.locationProvider = locationProvider; } @Override - public final String getLocalRack() + public String getLocalRack() { - return localRack; + return locationProvider.initialLocation().rack; } @Override - public final String getLocalDatacenter() + public String getLocalDatacenter() { - return localDc; - } - - @Override - public final String getRack(InetAddressAndPort endpoint) - { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return getLocalRack(); - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return DEFAULT_RACK; - return metadata.directory.location(nodeId).rack; - } - - @Override - public final String getDatacenter(InetAddressAndPort endpoint) - { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return getLocalDatacenter(); - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return DEFAULT_DC; - return metadata.directory.location(nodeId).datacenter; + return locationProvider.initialLocation().datacenter; } } diff --git a/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java b/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java index 2e7408be20..d2f4d88e17 100644 --- a/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java +++ b/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java @@ -18,12 +18,30 @@ package org.apache.cassandra.locator; import com.google.common.collect.Iterables; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.tcm.membership.Location; + +/** + * @deprecated + */ +@Deprecated(since = "CEP-21") public abstract class AbstractEndpointSnitch implements IEndpointSnitch { public abstract int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2); + @Override + public String getRack(InetAddressAndPort endpoint) + { + throw new UnsupportedOperationException("IEndpointSnitch has been deprecated and is no longer in use"); + } + + @Override + public String getDatacenter(InetAddressAndPort endpoint) + { + throw new UnsupportedOperationException("IEndpointSnitch has been deprecated and is no longer in use"); + } + /** * Sorts the Collection of node addresses by proximity to the given address * @param address the address to sort by proximity to @@ -53,7 +71,8 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch private boolean hasRemoteNode(ReplicaCollection l) { - String localDc = DatabaseDescriptor.getLocalDataCenter(); - return Iterables.any(l, replica -> !localDc.equals(getDatacenter(replica))); + Locator locator = DatabaseDescriptor.getLocator(); + Location local = locator.local(); + return Iterables.any(l, replica -> !local.sameDatacenter(locator.location(replica.endpoint()))); } } diff --git a/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java b/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java index 08c41f097f..b6901e2766 100644 --- a/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java +++ b/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java @@ -20,48 +20,15 @@ package org.apache.cassandra.locator; /** * An endpoint snitch tells Cassandra information about network topology that it can use to route * requests more efficiently. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public abstract class AbstractNetworkTopologySnitch extends AbstractEndpointSnitch { - /** - * Return the rack for which an endpoint resides in - * @param endpoint a specified endpoint - * @return string of rack - */ - abstract public String getRack(InetAddressAndPort endpoint); - - /** - * Return the data center for which an endpoint resides in - * @param endpoint a specified endpoint - * @return string of data center - */ - abstract public String getDatacenter(InetAddressAndPort endpoint); - + private static final NodeProximity proximity = new NetworkTopologyProximity(); @Override public int compareEndpoints(InetAddressAndPort address, Replica r1, Replica r2) { - InetAddressAndPort a1 = r1.endpoint(); - InetAddressAndPort a2 = r2.endpoint(); - if (address.equals(a1) && !address.equals(a2)) - return -1; - if (address.equals(a2) && !address.equals(a1)) - return 1; - - String addressDatacenter = getDatacenter(address); - String a1Datacenter = getDatacenter(a1); - String a2Datacenter = getDatacenter(a2); - if (addressDatacenter.equals(a1Datacenter) && !addressDatacenter.equals(a2Datacenter)) - return -1; - if (addressDatacenter.equals(a2Datacenter) && !addressDatacenter.equals(a1Datacenter)) - return 1; - - String addressRack = getRack(address); - String a1Rack = getRack(a1); - String a2Rack = getRack(a2); - if (addressRack.equals(a1Rack) && !addressRack.equals(a2Rack)) - return -1; - if (addressRack.equals(a2Rack) && !addressRack.equals(a1Rack)) - return 1; - return 0; + return proximity.compareEndpoints(address, r1, r2); } } diff --git a/src/java/org/apache/cassandra/locator/AlibabaCloudLocationProvider.java b/src/java/org/apache/cassandra/locator/AlibabaCloudLocationProvider.java new file mode 100644 index 0000000000..a3232cc296 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/AlibabaCloudLocationProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.IOException; + +import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; + +import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; + +public class AlibabaCloudLocationProvider extends CloudMetadataLocationProvider +{ + static final String DEFAULT_METADATA_SERVICE_URL = "http://100.100.100.200"; + static final String ZONE_NAME_QUERY_URL = "/latest/meta-data/zone-id"; + + /** + * Used via reflection by DatabaseDescriptor::createInitialLocationProvider + */ + public AlibabaCloudLocationProvider() throws IOException + { + this(new SnitchProperties()); + } + + public AlibabaCloudLocationProvider(SnitchProperties properties) throws IOException + { + this(new DefaultCloudMetadataServiceConnector(properties.putIfAbsent(METADATA_URL_PROPERTY, + DEFAULT_METADATA_SERVICE_URL))); + } + + public AlibabaCloudLocationProvider(AbstractCloudMetadataServiceConnector connector) throws IOException + { + super(connector, c -> SnitchUtils.parseLocation(c.apiCall(ZONE_NAME_QUERY_URL), c.getProperties().getDcSuffix())); + } +} diff --git a/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java b/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java index a7a9df4d1e..61af63ffea 100644 --- a/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java +++ b/src/java/org/apache/cassandra/locator/AlibabaCloudSnitch.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; +import static org.apache.cassandra.locator.AlibabaCloudLocationProvider.DEFAULT_METADATA_SERVICE_URL; /** * A snitch that assumes an ECS region is a DC and an ECS availability_zone @@ -29,12 +30,11 @@ import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector * format of the zone-id is like 'cn-hangzhou-a' where cn means china, hangzhou * means the hangzhou region, a means the az id. We use 'cn-hangzhou' as the dc, * and 'a' as the zone-id. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class AlibabaCloudSnitch extends AbstractCloudMetadataServiceSnitch { - static final String DEFAULT_METADATA_SERVICE_URL = "http://100.100.100.200"; - static final String ZONE_NAME_QUERY_URL = "/latest/meta-data/zone-id"; - public AlibabaCloudSnitch() throws IOException { this(new SnitchProperties()); @@ -48,7 +48,6 @@ public class AlibabaCloudSnitch extends AbstractCloudMetadataServiceSnitch public AlibabaCloudSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException { - super(connector, SnitchUtils.parseDcAndRack(connector.apiCall(ZONE_NAME_QUERY_URL), - connector.getProperties().getDcSuffix())); + super(new AlibabaCloudLocationProvider(connector)); } } diff --git a/src/java/org/apache/cassandra/locator/AzureCloudLocationProvider.java b/src/java/org/apache/cassandra/locator/AzureCloudLocationProvider.java new file mode 100644 index 0000000000..930a1f139b --- /dev/null +++ b/src/java/org/apache/cassandra/locator/AzureCloudLocationProvider.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.IOException; + +import com.google.common.collect.ImmutableMap; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.utils.JsonUtils; + +import static java.lang.String.format; +import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; + +public class AzureCloudLocationProvider extends CloudMetadataLocationProvider +{ + static final String DEFAULT_METADATA_SERVICE_URL = "http://169.254.169.254"; + static final String METADATA_QUERY_TEMPLATE = "/metadata/instance/compute?api-version=%s&format=json"; + static final String METADATA_HEADER = "Metadata"; + static final String API_VERSION_PROPERTY_KEY = "azure_api_version"; + static final String DEFAULT_API_VERSION = "2021-12-13"; + + /** + * Used via reflection by DatabaseDescriptor::createInitialLocationProvider + */ + public AzureCloudLocationProvider() throws IOException + { + this(new SnitchProperties()); + } + + public AzureCloudLocationProvider(SnitchProperties properties) throws IOException + { + this(new DefaultCloudMetadataServiceConnector(properties.putIfAbsent(METADATA_URL_PROPERTY, + DEFAULT_METADATA_SERVICE_URL))); + } + + public AzureCloudLocationProvider(AbstractCloudMetadataServiceConnector connector) throws IOException + { + super(connector, AzureCloudLocationProvider::resolveLocation); + } + + static Location resolveLocation(AbstractCloudMetadataServiceConnector connector) throws IOException + { + String apiVersion = connector.getProperties().get(API_VERSION_PROPERTY_KEY, DEFAULT_API_VERSION); + String response = connector.apiCall(format(METADATA_QUERY_TEMPLATE, apiVersion), + ImmutableMap.of(METADATA_HEADER, "true")); + JsonNode jsonNode = JsonUtils.JSON_OBJECT_MAPPER.readTree(response); + + JsonNode location = jsonNode.get("location"); + JsonNode zone = jsonNode.get("zone"); + JsonNode platformFaultDomain = jsonNode.get("platformFaultDomain"); + + String datacenter; + String rack; + + if (location == null || location.isNull() || location.asText().isEmpty()) + throw new ConfigurationException("Unable to retrieve initial location from cloud metadata service. " + + "This is required for registration, please check configuration"); + else + datacenter = location.asText(); + + if (zone == null || zone.isNull() || zone.asText().isEmpty()) + { + if (platformFaultDomain == null || platformFaultDomain.isNull() || platformFaultDomain.asText().isEmpty()) + { + throw new ConfigurationException("Unable to retrieve initial zone or platform fault domain from cloud metadata service. " + + "This is required for registration, please check configuration"); + } + else + { + rack = platformFaultDomain.asText(); + } + } + else + { + rack = zone.asText(); + } + + return new Location(datacenter + connector.getProperties().getDcSuffix(), "rack-" + rack); + } +} diff --git a/src/java/org/apache/cassandra/locator/AzureSnitch.java b/src/java/org/apache/cassandra/locator/AzureSnitch.java index 70729e1f3b..ae05de625a 100644 --- a/src/java/org/apache/cassandra/locator/AzureSnitch.java +++ b/src/java/org/apache/cassandra/locator/AzureSnitch.java @@ -20,15 +20,10 @@ package org.apache.cassandra.locator; import java.io.IOException; -import com.google.common.collect.ImmutableMap; - -import com.fasterxml.jackson.databind.JsonNode; import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; -import org.apache.cassandra.utils.JsonUtils; -import org.apache.cassandra.utils.Pair; -import static java.lang.String.format; import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; +import static org.apache.cassandra.locator.AzureCloudLocationProvider.DEFAULT_METADATA_SERVICE_URL; /** * AzureSnitch will resolve datacenter and rack by calling {@code /metadata/instance/compute} endpoint returning @@ -38,15 +33,11 @@ import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector * A datacenter is resolved from {@code location} field and a rack is resolved by looking into {@code zone} field first. * When zone is not set, or it is empty string, it will look into {@code platformFaultDomain} field. Such resolved * value is prepended by {@code rack-} string. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class AzureSnitch extends AbstractCloudMetadataServiceSnitch { - static final String DEFAULT_METADATA_SERVICE_URL = "http://169.254.169.254"; - static final String METADATA_QUERY_TEMPLATE = "/metadata/instance/compute?api-version=%s&format=json"; - static final String METADATA_HEADER = "Metadata"; - static final String API_VERSION_PROPERTY_KEY = "azure_api_version"; - static final String DEFAULT_API_VERSION = "2021-12-13"; - public AzureSnitch() throws IOException { this(new SnitchProperties()); @@ -60,43 +51,6 @@ public class AzureSnitch extends AbstractCloudMetadataServiceSnitch public AzureSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException { - super(connector, resolveDcAndRack(connector)); - } - - private static Pair resolveDcAndRack(AbstractCloudMetadataServiceConnector connector) throws IOException - { - String apiVersion = connector.getProperties().get(API_VERSION_PROPERTY_KEY, DEFAULT_API_VERSION); - String response = connector.apiCall(format(METADATA_QUERY_TEMPLATE, apiVersion), ImmutableMap.of(METADATA_HEADER, "true")); - JsonNode jsonNode = JsonUtils.JSON_OBJECT_MAPPER.readTree(response); - - JsonNode location = jsonNode.get("location"); - JsonNode zone = jsonNode.get("zone"); - JsonNode platformFaultDomain = jsonNode.get("platformFaultDomain"); - - String datacenter; - String rack; - - if (location == null || location.isNull() || location.asText().isEmpty()) - datacenter = DEFAULT_DC; - else - datacenter = location.asText(); - - if (zone == null || zone.isNull() || zone.asText().isEmpty()) - { - if (platformFaultDomain == null || platformFaultDomain.isNull() || platformFaultDomain.asText().isEmpty()) - { - rack = DEFAULT_RACK; - } - else - { - rack = platformFaultDomain.asText(); - } - } - else - { - rack = zone.asText(); - } - - return Pair.create(datacenter + connector.getProperties().getDcSuffix(), "rack-" + rack); + super(new AzureCloudLocationProvider(connector)); } } diff --git a/src/java/org/apache/cassandra/locator/BaseProximity.java b/src/java/org/apache/cassandra/locator/BaseProximity.java new file mode 100644 index 0000000000..928213bfe9 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/BaseProximity.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.tcm.membership.Location; + +public abstract class BaseProximity implements NodeProximity +{ + public abstract int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2); + + /** + * Sorts the Collection of node addresses by proximity to the given address + * @param address the address to sort by proximity to + * @param unsortedAddress the nodes to sort + * @return a new sorted List + */ + public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddress) + { + return unsortedAddress.sorted((r1, r2) -> compareEndpoints(address, r1, r2)); + } + + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) + { + // Querying remote DC is likely to be an order of magnitude slower than + // querying locally, so 2 queries to local nodes is likely to still be + // faster than 1 query involving remote ones + boolean mergedHasRemote = hasRemoteNode(merged); + return mergedHasRemote + ? hasRemoteNode(l1) || hasRemoteNode(l2) + : true; + } + + private boolean hasRemoteNode(ReplicaCollection l) + { + Locator locator = DatabaseDescriptor.getLocator(); + Location local = locator.local(); + return Iterables.any(l, replica -> !local.sameDatacenter(locator.location(replica.endpoint()))); + } +} diff --git a/src/java/org/apache/cassandra/locator/CloudMetadataLocationProvider.java b/src/java/org/apache/cassandra/locator/CloudMetadataLocationProvider.java new file mode 100644 index 0000000000..67cf172848 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/CloudMetadataLocationProvider.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.tcm.membership.Location; + +import static java.lang.String.format; + +public class CloudMetadataLocationProvider implements InitialLocationProvider +{ + static final Logger logger = LoggerFactory.getLogger(CloudMetadataLocationProvider.class); + + protected final AbstractCloudMetadataServiceConnector connector; + private final LocationResolver locationResolver; + private volatile Location location; + + public CloudMetadataLocationProvider(AbstractCloudMetadataServiceConnector connector, LocationResolver locationResolver) + { + this.connector = connector; + this.locationResolver = locationResolver; + } + + @Override + public final Location initialLocation() + { + if (location == null) + { + try + { + location = locationResolver.resolve(connector); + logger.info(format("%s using datacenter: %s, rack: %s, connector: %s, properties: %s", + getClass().getName(), location.datacenter, location.rack, connector, connector.getProperties())); + } + catch (IOException e) + { + throw new ConfigurationException("Unable to resolve initial location using cloud metadata service connector", e); + } + } + return location; + } + + public interface LocationResolver + { + Location resolve(AbstractCloudMetadataServiceConnector connector) throws IOException; + } +} diff --git a/src/java/org/apache/cassandra/locator/CloudstackLocationProvider.java b/src/java/org/apache/cassandra/locator/CloudstackLocationProvider.java new file mode 100644 index 0000000000..840aa06ded --- /dev/null +++ b/src/java/org/apache/cassandra/locator/CloudstackLocationProvider.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.URI; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileReader; +import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.utils.JVMStabilityInspector; + +import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; + +public class CloudstackLocationProvider extends CloudMetadataLocationProvider +{ + private static final Logger logger = LoggerFactory.getLogger(CloudstackLocationProvider.class); + static final String ZONE_NAME_QUERY_URI = "/latest/meta-data/availability-zone"; + + private static final String[] LEASE_FILES = + { + "file:///var/lib/dhcp/dhclient.eth0.leases", + "file:///var/lib/dhclient/dhclient.eth0.leases" + }; + + /** + * Used via reflection by DatabaseDescriptor::createInitialLocationProvider + */ + public CloudstackLocationProvider() throws IOException + { + this(new SnitchProperties(new Properties())); + } + + public CloudstackLocationProvider(SnitchProperties snitchProperties) throws IOException + { + this(new DefaultCloudMetadataServiceConnector(snitchProperties.putIfAbsent(METADATA_URL_PROPERTY, csMetadataEndpoint()))); + } + + public CloudstackLocationProvider(AbstractCloudMetadataServiceConnector connector) throws IOException + { + super(connector, CloudstackLocationProvider::resolveLocation); + logger.warn("{} is deprecated and not actively maintained. It will be removed in the next " + + "major version of Cassandra.", CloudstackSnitch.class.getName()); + } + + private static Location resolveLocation(AbstractCloudMetadataServiceConnector connector) throws IOException + { + String zone = connector.apiCall(ZONE_NAME_QUERY_URI); + String[] zoneParts = zone.split("-"); + + if (zoneParts.length != 3) + throw new ConfigurationException("CloudstackSnitch cannot handle invalid zone format: " + zone); + + return new Location(zoneParts[0] + '-' + zoneParts[1], zoneParts[2]); + } + + private static String csMetadataEndpoint() throws ConfigurationException + { + for (String lease_uri : LEASE_FILES) + { + try + { + File lease_file = new File(new URI(lease_uri)); + if (lease_file.exists()) + { + return csEndpointFromLease(lease_file); + } + } + catch (Exception e) + { + JVMStabilityInspector.inspectThrowable(e); + } + } + + throw new ConfigurationException("No valid DHCP lease file could be found."); + } + + private static String csEndpointFromLease(File lease) throws ConfigurationException + { + String line; + String endpoint = null; + Pattern identifierPattern = Pattern.compile("^[ \t]*option dhcp-server-identifier (.*);$"); + + try (BufferedReader reader = new BufferedReader(new FileReader(lease))) + { + + while ((line = reader.readLine()) != null) + { + Matcher matcher = identifierPattern.matcher(line); + + if (matcher.find()) + { + endpoint = matcher.group(1); + break; + } + } + } + catch (Exception e) + { + throw new ConfigurationException("CloudstackSnitch cannot access lease file."); + } + + if (endpoint == null) + { + throw new ConfigurationException("No metadata server could be found in lease file."); + } + + return "http://" + endpoint; + } +} diff --git a/src/java/org/apache/cassandra/locator/CloudstackSnitch.java b/src/java/org/apache/cassandra/locator/CloudstackSnitch.java index d5cebb69bf..bb0772ea5b 100644 --- a/src/java/org/apache/cassandra/locator/CloudstackSnitch.java +++ b/src/java/org/apache/cassandra/locator/CloudstackSnitch.java @@ -17,21 +17,8 @@ */ package org.apache.cassandra.locator; -import java.io.BufferedReader; import java.io.IOException; -import java.net.URI; import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileReader; -import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; -import org.apache.cassandra.utils.JVMStabilityInspector; -import org.apache.cassandra.utils.Pair; - -import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; /** * A snitch that assumes a Cloudstack Zone follows the typical convention @@ -45,14 +32,6 @@ import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector @Deprecated(since = "5.0") public class CloudstackSnitch extends AbstractCloudMetadataServiceSnitch { - static final String ZONE_NAME_QUERY_URI = "/latest/meta-data/availability-zone"; - - private static final String[] LEASE_FILES = - { - "file:///var/lib/dhcp/dhclient.eth0.leases", - "file:///var/lib/dhclient/dhclient.eth0.leases" - }; - public CloudstackSnitch() throws IOException { this(new SnitchProperties(new Properties())); @@ -60,78 +39,15 @@ public class CloudstackSnitch extends AbstractCloudMetadataServiceSnitch public CloudstackSnitch(SnitchProperties snitchProperties) throws IOException { - this(new DefaultCloudMetadataServiceConnector(snitchProperties.putIfAbsent(METADATA_URL_PROPERTY, csMetadataEndpoint()))); - } - - public CloudstackSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException - { - super(connector, resolveDcAndRack(connector)); + super( new CloudstackLocationProvider(snitchProperties)); logger.warn("{} is deprecated and not actively maintained. It will be removed in the next " + "major version of Cassandra.", CloudstackSnitch.class.getName()); } - private static Pair resolveDcAndRack(AbstractCloudMetadataServiceConnector connector) throws IOException + public CloudstackSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException { - String zone = connector.apiCall(ZONE_NAME_QUERY_URI); - String[] zoneParts = zone.split("-"); - - if (zoneParts.length != 3) - throw new ConfigurationException("CloudstackSnitch cannot handle invalid zone format: " + zone); - - return Pair.create(zoneParts[0] + '-' + zoneParts[1], zoneParts[2]); - } - - private static String csMetadataEndpoint() throws ConfigurationException - { - for (String lease_uri : LEASE_FILES) - { - try - { - File lease_file = new File(new URI(lease_uri)); - if (lease_file.exists()) - { - return csEndpointFromLease(lease_file); - } - } - catch (Exception e) - { - JVMStabilityInspector.inspectThrowable(e); - } - } - - throw new ConfigurationException("No valid DHCP lease file could be found."); - } - - private static String csEndpointFromLease(File lease) throws ConfigurationException - { - String line; - String endpoint = null; - Pattern identifierPattern = Pattern.compile("^[ \t]*option dhcp-server-identifier (.*);$"); - - try (BufferedReader reader = new BufferedReader(new FileReader(lease))) - { - - while ((line = reader.readLine()) != null) - { - Matcher matcher = identifierPattern.matcher(line); - - if (matcher.find()) - { - endpoint = matcher.group(1); - break; - } - } - } - catch (Exception e) - { - throw new ConfigurationException("CloudstackSnitch cannot access lease file."); - } - - if (endpoint == null) - { - throw new ConfigurationException("No metadata server could be found in lease file."); - } - - return "http://" + endpoint; + super(new CloudstackLocationProvider(connector)); + logger.warn("{} is deprecated and not actively maintained. It will be removed in the next " + + "major version of Cassandra.", CloudstackSnitch.class.getName()); } } diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java index bb652b6cff..1d3810613f 100644 --- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java +++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java @@ -47,8 +47,9 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_DYN /** * A dynamic snitch that sorts endpoints by latency with an adapted phi failure detector + * // TODO rename to DynamicNodeProximity or similar - but take care with jmx interface */ -public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements LatencySubscribers.Subscriber, DynamicEndpointSnitchMBean +public class DynamicEndpointSnitch implements NodeProximity, LatencySubscribers.Subscriber, DynamicEndpointSnitchMBean { private static final boolean USE_SEVERITY = !IGNORE_DYNAMIC_SNITCH_SEVERITY.getBoolean(); @@ -69,7 +70,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat private volatile HashMap scores = new HashMap<>(); private final ConcurrentHashMap samples = new ConcurrentHashMap<>(); - public final IEndpointSnitch subsnitch; + public final NodeProximity delegate; private volatile ScheduledFuture updateSchedular; private volatile ScheduledFuture resetSchedular; @@ -77,33 +78,21 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat private final Runnable update; private final Runnable reset; - public DynamicEndpointSnitch(IEndpointSnitch snitch) + public DynamicEndpointSnitch(NodeProximity delegate) { - this(snitch, null); + this(delegate, null); } - public DynamicEndpointSnitch(IEndpointSnitch snitch, String instance) + public DynamicEndpointSnitch(NodeProximity delegate, String instance) { mbeanName = "org.apache.cassandra.db:type=DynamicEndpointSnitch"; if (instance != null) mbeanName += ",instance=" + instance; - subsnitch = snitch; - update = new Runnable() - { - public void run() - { - updateScores(); - } - }; - reset = new Runnable() - { - public void run() - { - // we do this so that a host considered bad has a chance to recover, otherwise would we never try - // to read from it, which would cause its score to never change - reset(); - } - }; + this.delegate = delegate; + update = this::updateScores; + // we do this so that a host considered bad has a chance to recover, otherwise would we never try + // to read from it, which would cause its score to never change + reset = this::reset; if (DatabaseDescriptor.isDaemonInitialized()) { @@ -155,22 +144,6 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat MBeanWrapper.instance.unregisterMBean(mbeanName); } - @Override - public void gossiperStarting() - { - subsnitch.gossiperStarting(); - } - - public String getRack(InetAddressAndPort endpoint) - { - return subsnitch.getRack(endpoint); - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - return subsnitch.getDatacenter(endpoint); - } - @Override public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddresses) { @@ -196,7 +169,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat return replicas; // TODO: avoid copy - replicas = subsnitch.sortedByProximity(address, replicas); + replicas = delegate.sortedByProximity(address, replicas); HashMap scores = this.scores; // Make sure the score don't change in the middle of the loop below // (which wouldn't really matter here but its cleaner that way). ArrayList subsnitchOrderedScores = new ArrayList<>(replicas.size()); @@ -250,7 +223,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat } if (scored1.equals(scored2)) - return subsnitch.compareEndpoints(target, a1, a2); + return delegate.compareEndpoints(target, a1, a2); if (scored1 < scored2) return -1; else @@ -353,7 +326,10 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat public String getSubsnitchClassName() { - return subsnitch.getClass().getName(); + Class clazz = delegate.getClass(); + if (delegate instanceof SnitchAdapter) + clazz = ((SnitchAdapter)delegate).snitch.getClass(); + return clazz.getName(); } public List dumpTimings(String hostname) throws UnknownHostException @@ -401,7 +377,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) { - if (!subsnitch.isWorthMergingForRangeQuery(merged, l1, l2)) + if (!delegate.isWorthMergingForRangeQuery(merged, l1, l2)) return false; // skip checking scores in the single-node case @@ -433,9 +409,4 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat } return maxScore; } - - public boolean validate(Set datacenters, Set racks) - { - return subsnitch.validate(datacenters, racks); - } } diff --git a/src/java/org/apache/cassandra/locator/Ec2LocationProvider.java b/src/java/org/apache/cassandra/locator/Ec2LocationProvider.java new file mode 100644 index 0000000000..b6a3996df0 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/Ec2LocationProvider.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; + +public class Ec2LocationProvider extends CloudMetadataLocationProvider +{ + static final String SNITCH_PROP_NAMING_SCHEME = "ec2_naming_scheme"; + static final String EC2_NAMING_LEGACY = "legacy"; + static final String EC2_NAMING_STANDARD = "standard"; + + @VisibleForTesting + public static final String ZONE_NAME_QUERY = "/latest/meta-data/placement/availability-zone"; + + private final boolean usingLegacyNaming; + + /** + * Used via reflection by DatabaseDescriptor::createInitialLocationProvider + */ + public Ec2LocationProvider() throws IOException, ConfigurationException + { + this(new SnitchProperties()); + } + + public Ec2LocationProvider(SnitchProperties props) throws IOException, ConfigurationException + { + this(Ec2MetadataServiceConnector.create(props)); + } + + Ec2LocationProvider(AbstractCloudMetadataServiceConnector connector) throws IOException + { + super(connector, Ec2LocationProvider::getLocation); + usingLegacyNaming = isUsingLegacyNaming(connector.getProperties()); + } + + private static Location getLocation(AbstractCloudMetadataServiceConnector connector) throws IOException + { + // if using the full naming scheme, region name is created by removing letters from the + // end of the availability zone and zone is the full zone name + boolean usingLegacyNaming = isUsingLegacyNaming(connector.getProperties()); + + String az = connector.apiCall(ZONE_NAME_QUERY); + String region; + String localDc; + String localRack; + if (usingLegacyNaming) + { + // Split "us-east-1a" or "asia-1a" into "us-east"/"1a" and "asia"/"1a". + String[] splits = az.split("-"); + localRack = splits[splits.length - 1]; + + // hack for CASSANDRA-4026 + region = az.substring(0, az.length() - 1); + if (region.endsWith("1")) + region = az.substring(0, az.length() - 3); + } + else + { + // grab the region name, which is embedded in the availability zone name. + // thus an AZ of "us-east-1a" yields the region name "us-east-1" + region = az.replaceFirst("[a-z]+$",""); + localRack = az; + } + + localDc = region.concat(connector.getProperties().getDcSuffix()); + + return new Location(localDc, localRack); + } + + private static boolean isUsingLegacyNaming(SnitchProperties props) + { + return props.get(SNITCH_PROP_NAMING_SCHEME, EC2_NAMING_STANDARD).equalsIgnoreCase(EC2_NAMING_LEGACY); + } + + public void validate(ClusterMetadata metadata) + { + // Validate that the settings here match what was used to locate + // and register other nodes in the cluster (if there are any) + Set datacenters = metadata.directory.allDatacenterRacks().keySet(); + Set racks = new HashSet<>(); + for (String dc : datacenters) + racks.addAll(metadata.directory.datacenterRacks(dc).keySet()); + validate(datacenters, racks); + } + + public boolean validate(Set datacenters, Set racks) + { + boolean valid = true; + + for (String dc : datacenters) + { + // predicated on the late-2017 AWS naming 'convention' that all region names end with a digit. + // Unfortunately, life isn't that simple. Since we allow custom datacenter suffixes (CASSANDRA-5155), + // an operator could conceiveably make the suffix "a", and thus create a region name that looks just like + // one of the region's availability zones. (for example, "us-east-1a"). + // Further, we can't make any assumptions of what that suffix might be by looking at this node's + // datacenterSuffix as conceivably their could be many different suffixes in play for a given region. + // + // It is impossible to distinguish standard and legacy names for datacenters in some cases + // as the format didn't change for some regions (us-west-2 for example). + // We can still identify as legacy the dc names without a number as a suffix like us-east" + boolean dcUsesLegacyFormat = dc.matches("^[a-z]+-[a-z]+$"); + if (dcUsesLegacyFormat && !usingLegacyNaming) + { + valid = false; + break; + } + } + + for (String rack : racks) + { + // predicated on late-2017 AWS naming 'convention' that AZs do not have a digit as the first char - + // we had that in our legacy AZ (rack) names. Thus we test to see if the rack is in the legacy format. + // + // NOTE: the allowed custom suffix only applies to datacenter (region) names, not availability zones. + boolean rackUsesLegacyFormat = rack.matches("[\\d][a-z]"); + if (rackUsesLegacyFormat != usingLegacyNaming) + { + valid = false; + break; + } + } + + if (!valid) + { + throw new ConfigurationException(String.format("This ec2-enabled location provider appears to be using the " + + "%s naming scheme for regions, but existing nodes in cluster " + + "are using the opposite: " + + "region(s) = %s, availability zone(s) = %s. " + + "Please check the %s property in the %s configuration file " + + "for more details.", + usingLegacyNaming ? "legacy" : "standard", datacenters, racks, + SNITCH_PROP_NAMING_SCHEME, SnitchProperties.RACKDC_PROPERTY_FILENAME)); + } + return true; + } +} diff --git a/src/java/org/apache/cassandra/locator/Ec2MultiRegionAddressConfig.java b/src/java/org/apache/cassandra/locator/Ec2MultiRegionAddressConfig.java new file mode 100644 index 0000000000..bf2e83dfff --- /dev/null +++ b/src/java/org/apache/cassandra/locator/Ec2MultiRegionAddressConfig.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; + +import com.google.common.annotations.VisibleForTesting; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.utils.FBUtilities; + +public class Ec2MultiRegionAddressConfig implements NodeAddressConfig +{ + private static final Logger logger = LoggerFactory.getLogger(Ec2MultiRegionAddressConfig.class); + + @VisibleForTesting + public static final String PUBLIC_IP_QUERY = "/latest/meta-data/public-ipv4"; + @VisibleForTesting + public static final String PRIVATE_IP_QUERY = "/latest/meta-data/local-ipv4"; + private final String localPublicAddress; + private final String localPrivateAddress; + + /** + * Used via reflection by DatabaseDescriptor::createAddressConfig + */ + public Ec2MultiRegionAddressConfig() throws IOException + { + this(Ec2MetadataServiceConnector.create(new SnitchProperties())); + } + + @VisibleForTesting + public Ec2MultiRegionAddressConfig(AbstractCloudMetadataServiceConnector connector) throws IOException + { + this.localPublicAddress = connector.apiCall(PUBLIC_IP_QUERY); + logger.info("EC2 multi region address config using publicIP as identifier: {}", localPublicAddress); + this.localPrivateAddress = connector.apiCall(PRIVATE_IP_QUERY); + } + + @Override + public void configureAddresses() + { + // use the Public IP to broadcast Address to other nodes. + try + { + InetAddress broadcastAddress = InetAddress.getByName(localPublicAddress); + DatabaseDescriptor.setBroadcastAddress(broadcastAddress); + if (DatabaseDescriptor.getBroadcastRpcAddress() == null) + { + logger.info("broadcast_rpc_address unset, broadcasting public IP as rpc_address: {}", localPublicAddress); + DatabaseDescriptor.setBroadcastRpcAddress(broadcastAddress); + } + } + catch (UnknownHostException e) + { + throw new ConfigurationException("Unable to obtain public address for node from cloud metadata service", e); + } + + try + { + InetAddress privateAddress = InetAddress.getByName(localPrivateAddress); + FBUtilities.setLocalAddress(privateAddress); + } + catch (UnknownHostException e) + { + throw new ConfigurationException("Unable to obtain private address for node from cloud metadata service", e); + } + } + + @Override + public boolean preferLocalConnections() + { + // Always prefer re-connecting on private addresses if in the same datacenter (i.e. region) + return true; + } +} diff --git a/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java b/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java index b9f9f2d167..2344793768 100644 --- a/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java +++ b/src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java @@ -18,16 +18,10 @@ package org.apache.cassandra.locator; import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.gms.ApplicationState; -import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.service.StorageService; /** * 1) Snitch will automatically set the public IP by querying the AWS API @@ -39,14 +33,17 @@ import org.apache.cassandra.service.StorageService; * * Operational: All the nodes in this cluster needs to be able to (modify the * Security group settings in AWS) communicate via Public IP's. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class Ec2MultiRegionSnitch extends Ec2Snitch { + + private final Ec2MultiRegionAddressConfig addressConfig; @VisibleForTesting static final String PUBLIC_IP_QUERY = "/latest/meta-data/public-ipv4"; @VisibleForTesting static final String PRIVATE_IP_QUERY = "/latest/meta-data/local-ipv4"; - private final String localPrivateAddress; public Ec2MultiRegionSnitch() throws IOException, ConfigurationException { @@ -58,36 +55,22 @@ public class Ec2MultiRegionSnitch extends Ec2Snitch this(Ec2MetadataServiceConnector.create(props)); } - Ec2MultiRegionSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException + @VisibleForTesting + public Ec2MultiRegionSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException { super(connector); - InetAddress localPublicAddress = InetAddress.getByName(connector.apiCall(PUBLIC_IP_QUERY)); - logger.info("EC2Snitch using publicIP as identifier: {}", localPublicAddress); - localPrivateAddress = connector.apiCall(PRIVATE_IP_QUERY); - // use the Public IP to broadcast Address to other nodes. - DatabaseDescriptor.setBroadcastAddress(localPublicAddress); - if (DatabaseDescriptor.getBroadcastRpcAddress() == null) - { - logger.info("broadcast_rpc_address unset, broadcasting public IP as rpc_address: {}", localPublicAddress); - DatabaseDescriptor.setBroadcastRpcAddress(localPublicAddress); - } + addressConfig = new Ec2MultiRegionAddressConfig(connector); } @Override - public void gossiperStarting() + public void configureAddresses() { - super.gossiperStarting(); - InetAddressAndPort address; - try - { - address = InetAddressAndPort.getByName(localPrivateAddress); - } - catch (UnknownHostException e) - { - throw new RuntimeException(e); - } - Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT, StorageService.instance.valueFactory.internalAddressAndPort(address)); - Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, StorageService.instance.valueFactory.internalIP(address.getAddress())); - Gossiper.instance.register(new ReconnectableSnitchHelper(this, getLocalDatacenter(), true)); + addressConfig.configureAddresses(); + } + + @Override + public boolean preferLocalConnections() + { + return addressConfig.preferLocalConnections(); } } diff --git a/src/java/org/apache/cassandra/locator/Ec2Snitch.java b/src/java/org/apache/cassandra/locator/Ec2Snitch.java index 9b64efa041..2d30c0bd3b 100644 --- a/src/java/org/apache/cassandra/locator/Ec2Snitch.java +++ b/src/java/org/apache/cassandra/locator/Ec2Snitch.java @@ -23,7 +23,6 @@ import java.util.Set; import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.utils.Pair; /** * A snitch that assumes an EC2 region is a DC and an EC2 availability_zone @@ -44,7 +43,9 @@ import org.apache.cassandra.utils.Pair; * to a user is {@link Ec2MetadataServiceConnector.V2Connector#AWS_EC2_METADATA_TOKEN_TTL_SECONDS_HEADER_PROPERTY} * which is by default set to {@link Ec2MetadataServiceConnector.V2Connector#MAX_TOKEN_TIME_IN_SECONDS}. TTL has * to be an integer from the range [30, 21600]. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class Ec2Snitch extends AbstractCloudMetadataServiceSnitch { private static final String SNITCH_PROP_NAMING_SCHEME = "ec2_naming_scheme"; @@ -54,8 +55,6 @@ public class Ec2Snitch extends AbstractCloudMetadataServiceSnitch @VisibleForTesting public static final String ZONE_NAME_QUERY = "/latest/meta-data/placement/availability-zone"; - private final boolean usingLegacyNaming; - public Ec2Snitch() throws IOException, ConfigurationException { this(new SnitchProperties()); @@ -68,103 +67,12 @@ public class Ec2Snitch extends AbstractCloudMetadataServiceSnitch Ec2Snitch(AbstractCloudMetadataServiceConnector connector) throws IOException { - super(connector, getDcAndRack(connector)); - usingLegacyNaming = isUsingLegacyNaming(connector.getProperties()); - } - - private static Pair getDcAndRack(AbstractCloudMetadataServiceConnector connector) throws IOException - { - String az = connector.apiCall(ZONE_NAME_QUERY); - - // if using the full naming scheme, region name is created by removing letters from the - // end of the availability zone and zone is the full zone name - boolean usingLegacyNaming = isUsingLegacyNaming(connector.getProperties()); - String region; - String localDc; - String localRack; - if (usingLegacyNaming) - { - // Split "us-east-1a" or "asia-1a" into "us-east"/"1a" and "asia"/"1a". - String[] splits = az.split("-"); - localRack = splits[splits.length - 1]; - - // hack for CASSANDRA-4026 - region = az.substring(0, az.length() - 1); - if (region.endsWith("1")) - region = az.substring(0, az.length() - 3); - } - else - { - // grab the region name, which is embedded in the availability zone name. - // thus an AZ of "us-east-1a" yields the region name "us-east-1" - region = az.replaceFirst("[a-z]+$",""); - localRack = az; - } - - localDc = region.concat(connector.getProperties().getDcSuffix()); - - return Pair.create(localDc, localRack); - } - - private static boolean isUsingLegacyNaming(SnitchProperties props) - { - return props.get(SNITCH_PROP_NAMING_SCHEME, EC2_NAMING_STANDARD).equalsIgnoreCase(EC2_NAMING_LEGACY); + super(new Ec2LocationProvider(connector)); } @Override public boolean validate(Set datacenters, Set racks) { - return validate(datacenters, racks, usingLegacyNaming); - } - - @VisibleForTesting - static boolean validate(Set datacenters, Set racks, boolean usingLegacyNaming) - { - boolean valid = true; - - for (String dc : datacenters) - { - // predicated on the late-2017 AWS naming 'convention' that all region names end with a digit. - // Unfortunately, life isn't that simple. Since we allow custom datacenter suffixes (CASSANDRA-5155), - // an operator could conceiveably make the suffix "a", and thus create a region name that looks just like - // one of the region's availability zones. (for example, "us-east-1a"). - // Further, we can't make any assumptions of what that suffix might be by looking at this node's - // datacenterSuffix as conceivably their could be many different suffixes in play for a given region. - // - // It is impossible to distinguish standard and legacy names for datacenters in some cases - // as the format didn't change for some regions (us-west-2 for example). - // We can still identify as legacy the dc names without a number as a suffix like us-east" - boolean dcUsesLegacyFormat = dc.matches("^[a-z]+-[a-z]+$"); - if (dcUsesLegacyFormat && !usingLegacyNaming) - { - valid = false; - break; - } - } - - for (String rack : racks) - { - // predicated on late-2017 AWS naming 'convention' that AZs do not have a digit as the first char - - // we had that in our legacy AZ (rack) names. Thus we test to see if the rack is in the legacy format. - // - // NOTE: the allowed custom suffix only applies to datacenter (region) names, not availability zones. - boolean rackUsesLegacyFormat = rack.matches("[\\d][a-z]"); - if (rackUsesLegacyFormat != usingLegacyNaming) - { - valid = false; - break; - } - } - - if (!valid) - { - logger.error("This ec2-enabled snitch appears to be using the {} naming scheme for regions, " + - "but existing nodes in cluster are using the opposite: region(s) = {}, availability zone(s) = {}. " + - "Please check the {} property in the {} configuration file for more details.", - usingLegacyNaming ? "legacy" : "standard", datacenters, racks, - SNITCH_PROP_NAMING_SCHEME, SnitchProperties.RACKDC_PROPERTY_FILENAME); - } - - return valid; + return ((Ec2LocationProvider)locationProvider).validate(datacenters, racks); } } diff --git a/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java b/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java index d836cd1806..467d68b66d 100644 --- a/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java +++ b/src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java @@ -22,6 +22,7 @@ import java.net.UnknownHostException; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.utils.MBeanWrapper; +@Deprecated(since = "CEP-21") public class EndpointSnitchInfo implements EndpointSnitchInfoMBean { public static void create() @@ -31,26 +32,28 @@ public class EndpointSnitchInfo implements EndpointSnitchInfoMBean public String getDatacenter(String host) throws UnknownHostException { - return DatabaseDescriptor.getEndpointSnitch().getDatacenter(InetAddressAndPort.getByName(host)); + return DatabaseDescriptor.getLocator().location(InetAddressAndPort.getByName(host)).datacenter; } public String getRack(String host) throws UnknownHostException { - return DatabaseDescriptor.getEndpointSnitch().getRack(InetAddressAndPort.getByName(host)); + return DatabaseDescriptor.getLocator().location(InetAddressAndPort.getByName(host)).rack; } public String getDatacenter() { - return DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); + return DatabaseDescriptor.getLocator().local().datacenter; } public String getRack() { - return DatabaseDescriptor.getEndpointSnitch().getLocalRack(); + return DatabaseDescriptor.getLocator().local().rack; } public String getSnitchName() { - return DatabaseDescriptor.getEndpointSnitch().getClass().getName(); + NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); + Class clazz = proximity instanceof SnitchAdapter ? ((SnitchAdapter)proximity).snitch.getClass() : proximity.getClass(); + return clazz.getName(); } } diff --git a/src/java/org/apache/cassandra/locator/EndpointSnitchInfoMBean.java b/src/java/org/apache/cassandra/locator/EndpointSnitchInfoMBean.java index d6a18ff786..addc6a8ca8 100644 --- a/src/java/org/apache/cassandra/locator/EndpointSnitchInfoMBean.java +++ b/src/java/org/apache/cassandra/locator/EndpointSnitchInfoMBean.java @@ -21,7 +21,10 @@ import java.net.UnknownHostException; /** * MBean exposing standard Snitch info + * + * @deprecated see CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public interface EndpointSnitchInfoMBean { /** diff --git a/src/java/org/apache/cassandra/locator/GoogleCloudLocationProvider.java b/src/java/org/apache/cassandra/locator/GoogleCloudLocationProvider.java new file mode 100644 index 0000000000..f01955e803 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/GoogleCloudLocationProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.IOException; + +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; + +import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; + +public class GoogleCloudLocationProvider extends CloudMetadataLocationProvider +{ + static final String DEFAULT_METADATA_SERVICE_URL = "http://metadata.google.internal"; + static final String ZONE_NAME_QUERY_URL = "/computeMetadata/v1/instance/zone"; + static final ImmutableMap HEADERS = ImmutableMap.of("Metadata-Flavor", "Google"); + + /** + * Used via reflection by DatabaseDescriptor::createInitialLocationProvider + */ + public GoogleCloudLocationProvider() throws IOException + { + this(new SnitchProperties()); + } + + public GoogleCloudLocationProvider(SnitchProperties properties) throws IOException + { + this(new DefaultCloudMetadataServiceConnector(properties.putIfAbsent(METADATA_URL_PROPERTY, + DEFAULT_METADATA_SERVICE_URL))); + } + + public GoogleCloudLocationProvider(AbstractCloudMetadataServiceConnector connector) throws IOException + { + super(connector, c -> SnitchUtils.parseLocation(c.apiCall(ZONE_NAME_QUERY_URL, HEADERS), + c.getProperties().getDcSuffix())); + } +} diff --git a/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java b/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java index af3257187e..fdd0991c23 100644 --- a/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java +++ b/src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java @@ -19,21 +19,14 @@ package org.apache.cassandra.locator; import java.io.IOException; -import com.google.common.collect.ImmutableMap; - -import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; - -import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; - /** * A snitch that assumes an GCE region is a DC and an GCE availability_zone * is a rack. This information is available in the config for the node. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class GoogleCloudSnitch extends AbstractCloudMetadataServiceSnitch { - static final String DEFAULT_METADATA_SERVICE_URL = "http://metadata.google.internal"; - static final String ZONE_NAME_QUERY_URL = "/computeMetadata/v1/instance/zone"; - public GoogleCloudSnitch() throws IOException { this(new SnitchProperties()); @@ -41,14 +34,11 @@ public class GoogleCloudSnitch extends AbstractCloudMetadataServiceSnitch public GoogleCloudSnitch(SnitchProperties properties) throws IOException { - this(new DefaultCloudMetadataServiceConnector(properties.putIfAbsent(METADATA_URL_PROPERTY, - DEFAULT_METADATA_SERVICE_URL))); + super(new GoogleCloudLocationProvider(properties)); } public GoogleCloudSnitch(AbstractCloudMetadataServiceConnector connector) throws IOException { - super(connector, SnitchUtils.parseDcAndRack(connector.apiCall(ZONE_NAME_QUERY_URL, - ImmutableMap.of("Metadata-Flavor", "Google")), - connector.getProperties().getDcSuffix())); + super(new GoogleCloudLocationProvider(connector)); } } diff --git a/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java b/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java index bbd110588b..2e02165906 100644 --- a/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java +++ b/src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java @@ -18,107 +18,40 @@ package org.apache.cassandra.locator; -import java.util.concurrent.atomic.AtomicReference; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.gms.ApplicationState; -import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.tcm.membership.Location; - -public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch// implements IEndpointStateChangeSubscriber +/** + * @deprecated See CASSANDRA-19488 + */ +@Deprecated(since = "CEP-21") +public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch { - private static final Logger logger = LoggerFactory.getLogger(GossipingPropertyFileSnitch.class); - - private final String myDC; - private final String myRack; - private final boolean preferLocal; - private final AtomicReference snitchHelperReference; - private static final String DEFAULT_DC = "UNKNOWN_DC"; - private static final String DEFAULT_RACK = "UNKNOWN_RACK"; + private final Location fromConfig; + public final boolean preferLocal; public GossipingPropertyFileSnitch() throws ConfigurationException { - SnitchProperties properties = loadConfiguration(); - - myDC = properties.get("dc", DEFAULT_DC).trim(); - myRack = properties.get("rack", DEFAULT_RACK).trim(); + SnitchProperties properties = RackDCFileLocationProvider.loadConfiguration(); + fromConfig = new RackDCFileLocationProvider(properties).initialLocation(); preferLocal = Boolean.parseBoolean(properties.get("prefer_local", "false")); - snitchHelperReference = new AtomicReference<>(); } - private static SnitchProperties loadConfiguration() throws ConfigurationException + @Override + public String getLocalRack() { - final SnitchProperties properties = new SnitchProperties(); - if (!properties.contains("dc") || !properties.contains("rack")) - throw new ConfigurationException("DC or rack not found in snitch properties, check your configuration in: " + SnitchProperties.RACKDC_PROPERTY_FILENAME); - - return properties; + return fromConfig.rack; } - /** - * Return the data center for which an endpoint resides in - * - * @param endpoint the endpoint to process - * @return string of data center - */ - public String getDatacenter(InetAddressAndPort endpoint) + @Override + public String getLocalDatacenter() { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return myDC; - - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return DEFAULT_DC; - return metadata.directory.location(nodeId).datacenter; + return fromConfig.datacenter; } - /** - * Return the rack for which an endpoint resides in - * - * @param endpoint the endpoint to process - * @return string of rack - */ - public String getRack(InetAddressAndPort endpoint) + @Override + public boolean preferLocalConnections() { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return myRack; - - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return DEFAULT_RACK; - return metadata.directory.location(nodeId).rack; - } - - public void gossiperStarting() - { - super.gossiperStarting(); - - Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT, - StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort())); - Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, - StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress())); - - loadGossiperState(); - } - - private void loadGossiperState() - { - assert Gossiper.instance != null; - - ReconnectableSnitchHelper pendingHelper = new ReconnectableSnitchHelper(this, myDC, preferLocal); - Gossiper.instance.register(pendingHelper); - - pendingHelper = snitchHelperReference.getAndSet(pendingHelper); - if (pendingHelper != null) - Gossiper.instance.unregister(pendingHelper); + return preferLocal; } } diff --git a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java index 0120391265..4d50336810 100644 --- a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java +++ b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java @@ -26,8 +26,9 @@ import org.apache.cassandra.utils.FBUtilities; * This interface helps determine location of node in the datacenter relative to another node. * Give a node A and another node B it can tell if A and B are on the same rack or in the same * datacenter. + * @deprecated Replaced by Locator and NodeProximity; see CASSANDRA-19488 */ - +@Deprecated(since="CEP-21") public interface IEndpointSnitch { /** @@ -94,4 +95,11 @@ public interface IEndpointSnitch { return true; } + + default void configureAddresses() {} + default boolean preferLocalConnections() + { + return false; + } } + diff --git a/src/java/org/apache/cassandra/locator/InOurDc.java b/src/java/org/apache/cassandra/locator/InOurDc.java index 34e8ef89c3..4859ebd23a 100644 --- a/src/java/org/apache/cassandra/locator/InOurDc.java +++ b/src/java/org/apache/cassandra/locator/InOurDc.java @@ -20,8 +20,8 @@ package org.apache.cassandra.locator; import java.util.function.Predicate; -import static org.apache.cassandra.config.DatabaseDescriptor.getEndpointSnitch; import static org.apache.cassandra.config.DatabaseDescriptor.getLocalDataCenter; +import static org.apache.cassandra.config.DatabaseDescriptor.getLocator; public class InOurDc { @@ -29,49 +29,44 @@ public class InOurDc private static EndpointTester endpoints; final String dc; - final IEndpointSnitch snitch; + final Locator locator; - private InOurDc(String dc, IEndpointSnitch snitch) + private InOurDc(String dc, Locator locator) { this.dc = dc; - this.snitch = snitch; + this.locator = locator; } boolean stale() { - return dc != getLocalDataCenter() - || snitch != getEndpointSnitch() - // this final clause checks if somehow the snitch/localDc have got out of whack; - // presently, this is possible but very unlikely, but this check will also help - // resolve races on these global fields as well - || !dc.equals(snitch.getLocalDatacenter()); + return !dc.equals(getLocalDataCenter()); } private static final class ReplicaTester extends InOurDc implements Predicate { - private ReplicaTester(String dc, IEndpointSnitch snitch) + private ReplicaTester(String dc, Locator locator) { - super(dc, snitch); + super(dc, locator); } @Override public boolean test(Replica replica) { - return dc.equals(snitch.getDatacenter(replica.endpoint())); + return dc.equals(locator.location(replica.endpoint()).datacenter); } } private static final class EndpointTester extends InOurDc implements Predicate { - private EndpointTester(String dc, IEndpointSnitch snitch) + private EndpointTester(String dc, Locator locator) { - super(dc, snitch); + super(dc, locator); } @Override public boolean test(InetAddressAndPort endpoint) { - return dc.equals(snitch.getDatacenter(endpoint)); + return dc.equals(locator.location(endpoint).datacenter); } } @@ -79,7 +74,7 @@ public class InOurDc { ReplicaTester cur = replicas; if (cur == null || cur.stale()) - replicas = cur = new ReplicaTester(getLocalDataCenter(), getEndpointSnitch()); + replicas = cur = new ReplicaTester(getLocalDataCenter(), getLocator()); return cur; } @@ -87,7 +82,7 @@ public class InOurDc { EndpointTester cur = endpoints; if (cur == null || cur.stale()) - endpoints = cur = new EndpointTester(getLocalDataCenter(), getEndpointSnitch()); + endpoints = cur = new EndpointTester(getLocalDataCenter(), getLocator()); return cur; } diff --git a/src/java/org/apache/cassandra/locator/InitialLocationProvider.java b/src/java/org/apache/cassandra/locator/InitialLocationProvider.java new file mode 100644 index 0000000000..17fafd88af --- /dev/null +++ b/src/java/org/apache/cassandra/locator/InitialLocationProvider.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; + +public interface InitialLocationProvider +{ + /** + * Provides the Location with which a new node should register itself with ClusterMetadata. + * After registration, this is no longer necessary and Location is always sourced from + * ClusterMetadata + * @return the datacenter and rack to register with + */ + Location initialLocation(); + + /** + * Validate that the locations of previously registered peers are considered valid by the locally configured + * InitialLocationProvider. In practice, of the in-tree implementations only Ec2LocationProvider overrides this and + * uses it to ensure that the same ec2 naming scheme is used across all peers. + * See CASSANDRA-7839 for origins. + * @param metadata ClusterMetadata at the time of registering + * @return true if the implementation considers the locations of existing nodes compatible with its own + * configuration, false otherwise + */ + default void validate(ClusterMetadata metadata) {} +} diff --git a/src/java/org/apache/cassandra/locator/LocationInfo.java b/src/java/org/apache/cassandra/locator/LocationInfo.java new file mode 100644 index 0000000000..c0b3e78818 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/LocationInfo.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.locator; + +import java.net.UnknownHostException; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.utils.MBeanWrapper; + +public class LocationInfo implements LocationInfoMBean +{ + public static void create() + { + MBeanWrapper.instance.registerMBean(new LocationInfo(), "org.apache.cassandra.db:type=LocationInfo"); + } + + public String getDatacenter(String host) throws UnknownHostException + { + return DatabaseDescriptor.getLocator().location(InetAddressAndPort.getByName(host)).datacenter; + } + + public String getRack(String host) throws UnknownHostException + { + return DatabaseDescriptor.getLocator().location(InetAddressAndPort.getByName(host)).rack; + } + + public String getDatacenter() + { + return DatabaseDescriptor.getLocator().local().datacenter; + } + + public String getRack() + { + return DatabaseDescriptor.getLocator().local().rack; + } + + public String getNodeProximityName() + { + NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); + Class clazz = proximity instanceof SnitchAdapter ? ((SnitchAdapter)proximity).snitch.getClass() : proximity.getClass(); + return clazz.getName(); + } + + public boolean hasLegacySnitchAdapter() + { + NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); + if (proximity instanceof DynamicEndpointSnitch) + return ((DynamicEndpointSnitch)proximity).delegate instanceof SnitchAdapter; + return proximity instanceof SnitchAdapter; + } +} diff --git a/src/java/org/apache/cassandra/locator/LocationInfoMBean.java b/src/java/org/apache/cassandra/locator/LocationInfoMBean.java new file mode 100644 index 0000000000..e1c7100eca --- /dev/null +++ b/src/java/org/apache/cassandra/locator/LocationInfoMBean.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.net.UnknownHostException; + +public interface LocationInfoMBean +{ + /** + * Provides the Rack name depending on the respective snitch used, given the host name/ip + * @param host + * @throws UnknownHostException + */ + public String getRack(String host) throws UnknownHostException; + + /** + * Provides the Datacenter name depending on the respective snitch used, given the hostname/ip + * @param host + * @throws UnknownHostException + */ + public String getDatacenter(String host) throws UnknownHostException; + + /** + * Provides the Rack name depending on the respective snitch used for this node + */ + public String getRack(); + + /** + * Provides the Datacenter name depending on the respective snitch used for this node + */ + public String getDatacenter(); + + public String getNodeProximityName(); + public boolean hasLegacySnitchAdapter(); +} diff --git a/src/java/org/apache/cassandra/locator/Locator.java b/src/java/org/apache/cassandra/locator/Locator.java new file mode 100644 index 0000000000..20ad604692 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/Locator.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.RegistrationStatus; +import org.apache.cassandra.tcm.membership.Directory; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Provides Location (datacenter & rack) information for endpoints. Usually this is obtained directly + * from ClusterMetadata.directory, using either a specific directory instance or the most current + * published one. During initial startup, location info for the local node may be derived from some + * other source, such as a config file or cloud metadata api. This is then used to register the node + * and its location in ClusterMetadata, which then becomes the ultimate source of truth. + */ +public class Locator +{ + private final InetAddressAndPort localEndpoint; + + // Indicates whether the *local node* is yet to register itself with ClusterMetadata. + // This is relevant because once a node is registered, it's location is always derived + // from ClusterMetadata. However, before registering the location is obtained from + // configuration. This pre-registration location is what the node supplies to ClusterMetadata + // when registering and is also used to determine DC locality of peers, for instance when + // establishing initial internode connections during the discovery phase. + private final RegistrationStatus state; + + // Source of truth for location lookups. This may be null, in which case every lookup will + // use the most up to date version from ClusterMetadata.current().directory + private final Directory directory; + + // Supplies the Location for this node only during its initial startup. This location will be + // used to register the node with ClusterMetadata and is not used after that has occurred. + // See DatabaseDescriptor::getInitialLocationProvider + private final InitialLocationProvider locationProvider; + + // This is the Location used to register this node during its initial startup. It is lazily initialized + // using the supplied InitialLocationProvider and memoized here as that may be a non-trivial operation. + // Some providers fetch location metadata from remote services etc. It should usually be unnecessary to + // access the initialization location after a node's first startup. + private volatile Location initializationLocation; + + // Set from ClusterMetadata. After initial registration has happened, location of this node itself + // is always taken from ClusterMetadata. + private volatile VersionedLocation local; + + private static class VersionedLocation + { + final Epoch epoch; + final Location location; + private VersionedLocation(Epoch epoch, Location location) + { + this.epoch = epoch; + this.location = location; + } + } + + public static Locator usingDirectory(Directory directory) + { + return new Locator(RegistrationStatus.instance, + FBUtilities.getBroadcastAddressAndPort(), + DatabaseDescriptor.getInitialLocationProvider(), + directory); + } + + public static Locator forClients() + { + return new Locator(RegistrationStatus.instance, + FBUtilities.getBroadcastAddressAndPort(), + () -> Location.UNKNOWN, + null); + } + + /** + * Creates a Locator instance which always uses the Directory from the most current ClusterMetadata. + * This means that the values returned from {@link this#location(InetAddressAndPort)} can change between + * invocations, if interleaved with the publication of updated cluster metadata. + */ + public Locator(RegistrationStatus state, + InetAddressAndPort localEndpoint, + InitialLocationProvider provider) + { + this(state, localEndpoint, provider, null); + } + + /** + * Creates a Locator instance which returns consistent results based on the supplied Directory instance. + * Changes to RegistrationStatus could still have an effect i.e. the node transitioned from UNREGISTERED to + * REGISTERED between two calls to local(), the first would return the Location according to + * DatabaseDescriptor.getInitialLocationProvider, but the second would consult the supplied Directory. + */ + public Locator(RegistrationStatus state, + InetAddressAndPort localEndpoint, + InitialLocationProvider provider, + Directory directory) + { + this.state = state; + this.localEndpoint = localEndpoint; + this.locationProvider = provider; + this.directory = directory; + this.local = new VersionedLocation(Epoch.EMPTY, Location.UNKNOWN); + } + + public Location location(InetAddressAndPort endpoint) + { + switch (state.getCurrent()) + { + case INITIAL: + return endpoint.equals(localEndpoint) ? initialLocation() : Location.UNKNOWN; + case UNREGISTERED: + return endpoint.equals(localEndpoint) ? initialLocation() : fromDirectory(endpoint); + default: + return fromDirectory(endpoint); + } + } + + public Location local() + { + switch (state.getCurrent()) + { + case INITIAL: + case UNREGISTERED: + return initialLocation(); + default: + // For now, local location is immutable and once registered with cluster metadata, it cannot be + // changed. Revisit this if that assumption changes. + VersionedLocation location = local; + if (location.epoch.isAfter(Epoch.EMPTY)) + return location.location; + + local = versionedFromDirectory(localEndpoint); + return local.location; + } + } + + // The distinction between versioned and unversioned may be removed if/when we allow + // a node's Location to be modified after registration. This duplication should not + // be necessary then. + private VersionedLocation versionedFromDirectory(InetAddressAndPort endpoint) + { + Directory source = directory; + if (source == null) + { + ClusterMetadata metadata = ClusterMetadata.currentNullable(); + if (metadata == null) + return new VersionedLocation(Epoch.EMPTY, Location.UNKNOWN); + source = metadata.directory; + } + NodeId nodeId = source.peerId(endpoint); + Location location = nodeId != null ? source.location(nodeId) : Location.UNKNOWN; + return new VersionedLocation(source.lastModified(), location); + } + + private Location fromDirectory(InetAddressAndPort endpoint) + { + Directory source = directory; + if (source == null) + { + ClusterMetadata metadata = ClusterMetadata.currentNullable(); + if (metadata == null) + return Location.UNKNOWN; + source = metadata.directory; + } + NodeId nodeId = source.peerId(endpoint); + return nodeId != null ? source.location(nodeId) : Location.UNKNOWN; + } + + private Location initialLocation() + { + if (initializationLocation == null) + initializationLocation = locationProvider.initialLocation(); + return initializationLocation; + } +} diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyProximity.java b/src/java/org/apache/cassandra/locator/NetworkTopologyProximity.java new file mode 100644 index 0000000000..eddcb36303 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/NetworkTopologyProximity.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.tcm.membership.Location; + +public class NetworkTopologyProximity extends BaseProximity +{ + public int compareEndpoints(InetAddressAndPort address, Replica r1, Replica r2) + { + InetAddressAndPort a1 = r1.endpoint(); + InetAddressAndPort a2 = r2.endpoint(); + if (address.equals(a1) && !address.equals(a2)) + return -1; + if (address.equals(a2) && !address.equals(a1)) + return 1; + + Locator locator = DatabaseDescriptor.getLocator(); + Location l1 = locator.location(a1); + Location l2 = locator.location(a2); + Location location = locator.location(address); + + if (location.datacenter.equals(l1.datacenter) && !location.datacenter.equals(l2.datacenter)) + return -1; + if (location.datacenter.equals(l2.datacenter) && !location.datacenter.equals(l1.datacenter)) + return 1; + + if (location.rack.equals(l1.rack) && !location.rack.equals(l2.rack)) + return -1; + if (location.rack.equals(l2.rack) && !location.rack.equals(l1.rack)) + return 1; + return 0; + } +} diff --git a/src/java/org/apache/cassandra/locator/NoOpProximity.java b/src/java/org/apache/cassandra/locator/NoOpProximity.java new file mode 100644 index 0000000000..342f12e619 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/NoOpProximity.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +public class NoOpProximity extends BaseProximity +{ + @Override + public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddress) + { + // Optimization to avoid walking the list + return unsortedAddress; + } + + @Override + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) + { + // Making all endpoints equal ensures we won't change the original ordering (since + // Collections.sort is guaranteed to be stable) + return 0; + } +} diff --git a/src/java/org/apache/cassandra/locator/NodeAddressConfig.java b/src/java/org/apache/cassandra/locator/NodeAddressConfig.java new file mode 100644 index 0000000000..23bf4e9976 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/NodeAddressConfig.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.config.DatabaseDescriptor; + +public interface NodeAddressConfig +{ + void configureAddresses(); + boolean preferLocalConnections(); + + NodeAddressConfig DEFAULT = new NodeAddressConfig() + { + @Override + public void configureAddresses() + { + // Default is to use whatever addresses are specified in node config already, so this is a no-op + } + + @Override + public boolean preferLocalConnections() + { + // Previously the equivalent config was hard coded for Ec2MultiRegionSnitch and read from + // cassandra-rackdc.properties in the case of GossipingPropertyFileSnitch. In legacy config + // mode, where one of those IEndpointSnitch impls is still specified the original behaviour + // is preserved. For modern config the option can be specified in the main yaml, distinct + // from the NodeProximity and InitialLocationProvider options. + return DatabaseDescriptor.preferLocalConnections(); + } + }; +} diff --git a/src/java/org/apache/cassandra/locator/NodeProximity.java b/src/java/org/apache/cassandra/locator/NodeProximity.java new file mode 100644 index 0000000000..cbb7158aaf --- /dev/null +++ b/src/java/org/apache/cassandra/locator/NodeProximity.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +public interface NodeProximity +{ + /** + * returns a new List sorted by proximity to the given endpoint + */ + public > C sortedByProximity(final InetAddressAndPort address, C addresses); + + /** + * compares two endpoints in relation to the target endpoint, returning as Comparator.compare would + */ + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2); + + /** + * Returns whether for a range query doing a query against merged is likely + * to be faster than 2 sequential queries, one against l1 followed by one against l2. + */ + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2); +} diff --git a/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java b/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java index 430779f468..ee70fa8ae4 100644 --- a/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java +++ b/src/java/org/apache/cassandra/locator/PropertyFileSnitch.java @@ -17,22 +17,8 @@ */ package org.apache.cassandra.locator; -import java.io.InputStream; -import java.net.UnknownHostException; -import java.util.Map; -import java.util.Properties; - -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.Location; -import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.utils.FBUtilities; - -import org.apache.commons.lang3.StringUtils; /** *

@@ -50,133 +36,28 @@ import org.apache.commons.lang3.StringUtils; * this is done automatically with location derived from gossip state (ultimately from system.local). * Once registered, the Rack & DC should not be changed but currently the only safeguards against this are the * StartupChecks which validate the snitch against system.local. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class PropertyFileSnitch extends AbstractNetworkTopologySnitch { - private static final Logger logger = LoggerFactory.getLogger(PropertyFileSnitch.class); - - public static final String SNITCH_PROPERTIES_FILENAME = "cassandra-topology.properties"; - // All the defaults - private static final String DEFAULT_PROPERTY = "default"; - @VisibleForTesting - public static final String DEFAULT_DC = "default"; - @VisibleForTesting - public static final String DEFAULT_RACK = "default"; - + // Used only during initialization of a new node. This provides the location it will register in cluster metadata private final Location local; - public PropertyFileSnitch() throws ConfigurationException { - local = loadConfiguration(); + local = new TopologyFileLocationProvider().initialLocation(); } - public String getDatacenter(InetAddressAndPort endpoint) + @Override + public String getLocalRack() { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return local.datacenter; - - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return DEFAULT_DC; - return metadata.directory.location(nodeId).datacenter; + return local.rack; } - /** - * Return the rack for which an endpoint resides in - * - * @param endpoint the endpoint to process - * @return string of rack - */ - public String getRack(InetAddressAndPort endpoint) + @Override + public String getLocalDatacenter() { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return local.rack; - - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return DEFAULT_RACK; - return metadata.directory.location(nodeId).rack; - } - - private Location makeLocation(String value) - { - if (value == null || value.isEmpty()) - return null; - - String[] parts = value.split(":"); - if (parts.length < 2) - { - return new Location(DEFAULT_DC, DEFAULT_RACK); - } - else - { - return new Location(parts[0].trim(), parts[1].trim()); - } - } - - private Location loadConfiguration() throws ConfigurationException - { - Properties properties = new Properties(); - try (InputStream stream = getClass().getClassLoader().getResourceAsStream(SNITCH_PROPERTIES_FILENAME)) - { - properties.load(stream); - } - catch (Exception e) - { - throw new ConfigurationException("Unable to read " + SNITCH_PROPERTIES_FILENAME, e); - } - - // may be null, which is ok unless config doesn't contain the location of the local node - Location defaultLocation = makeLocation(properties.getProperty(DEFAULT_PROPERTY)); - Location local = null; - InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); - for (Map.Entry entry : properties.entrySet()) - { - String key = (String) entry.getKey(); - String value = (String) entry.getValue(); - if (DEFAULT_PROPERTY.equals(key)) - continue; - - String hostString = StringUtils.remove(key, '/'); - try - { - InetAddressAndPort host = InetAddressAndPort.getByName(hostString); - if (host.equals(broadcastAddress)) - { - local = makeLocation(value); - break; - } - } - catch (UnknownHostException e) - { - throw new ConfigurationException("Unknown host " + hostString, e); - } - - } - - if (local == null) - { - if (defaultLocation == null) - { - throw new ConfigurationException(String.format("Snitch definitions at %s do not define a location for " + - "this node's broadcast address %s, nor does it provides a default", - SNITCH_PROPERTIES_FILENAME, broadcastAddress)); - } - else - { - logger.debug("Broadcast address {} was not present in snitch config, using default location {}. " + - "This only matters on first boot, before registering with the cluster metadata service", - broadcastAddress, defaultLocation); - return defaultLocation; - } - } - - logger.debug("Loaded location {} for broadcast address {} from property file. " + - "This only matters on first boot, before registering with the cluster metadata service", - local, broadcastAddress); - return local; + return local.datacenter; } } diff --git a/src/java/org/apache/cassandra/locator/RackDCFileLocationProvider.java b/src/java/org/apache/cassandra/locator/RackDCFileLocationProvider.java new file mode 100644 index 0000000000..6443de78d4 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/RackDCFileLocationProvider.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.tcm.membership.Location; + +/** + * Emulates the previous behaviour of GossipingPropertyFileSnitch; reads cassandra-rackdc.properties + * to identify the local datacenter and rack for one time use during node registration. + * + * Based on a properties file in the following format: + * + * dc=DC1 + * rack=RAC1 + */ +public class RackDCFileLocationProvider implements InitialLocationProvider +{ + private final Location local; + private static final Location DEFAULT = new Location("UNKNOWN_DC", "UNKNOWN_RACK"); + + /** + * Used via reflection by DatabaseDescriptor::createInitialLocationProvider + */ + public RackDCFileLocationProvider() + { + this(loadConfiguration()); + } + + /** + * Used in legacy compatibility mode by GossipingPropertyFileSnitch + * @param properties + */ + RackDCFileLocationProvider(SnitchProperties properties) + { + local = new Location(properties.get("dc", DEFAULT.datacenter).trim(), + properties.get("rack", DEFAULT.rack).trim()); + } + + @Override + public Location initialLocation() + { + return local; + } + + public static SnitchProperties loadConfiguration() throws ConfigurationException + { + final SnitchProperties properties = new SnitchProperties(); + if (!properties.contains("dc") || !properties.contains("rack")) + throw new ConfigurationException("DC or rack not found in snitch properties, check your configuration in: " + SnitchProperties.RACKDC_PROPERTY_FILENAME); + + return properties; + } + + +} diff --git a/src/java/org/apache/cassandra/locator/RackInferringSnitch.java b/src/java/org/apache/cassandra/locator/RackInferringSnitch.java index cc47e0e8fb..88487c333b 100644 --- a/src/java/org/apache/cassandra/locator/RackInferringSnitch.java +++ b/src/java/org/apache/cassandra/locator/RackInferringSnitch.java @@ -17,9 +17,7 @@ */ package org.apache.cassandra.locator; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.Location; -import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.utils.FBUtilities; /** @@ -29,7 +27,9 @@ import org.apache.cassandra.utils.FBUtilities; * Local location is derived from (broadcast) ip address and added to ClusterMetadata during node * registration. Every member of the cluster is required to do this, hence remote peers' Location * can always be retrieved, consistently. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class RackInferringSnitch extends AbstractNetworkTopologySnitch { final Location local; @@ -37,31 +37,25 @@ public class RackInferringSnitch extends AbstractNetworkTopologySnitch public RackInferringSnitch() { InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort(); - local = new Location(Integer.toString(localAddress.getAddress().getAddress()[1] & 0xFF, 10), - Integer.toString(localAddress.getAddress().getAddress()[2] & 0xFF, 10)); + local = inferLocation(localAddress); } - public String getDatacenter(InetAddressAndPort endpoint) + public static Location inferLocation(InetAddressAndPort address) { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return local.datacenter; + return new Location(Integer.toString(address.getAddress().getAddress()[1] & 0xFF, 10), + Integer.toString(address.getAddress().getAddress()[2] & 0xFF, 10)); - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return Integer.toString(endpoint.getAddress().getAddress()[1] & 0xFF, 10); - return metadata.directory.location(nodeId).datacenter; } + @Override public String getRack(InetAddressAndPort endpoint) { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return local.rack; + return inferLocation(endpoint).rack; + } - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId nodeId = metadata.directory.peerId(endpoint); - if (nodeId == null) - return Integer.toString(endpoint.getAddress().getAddress()[2] & 0xFF, 10); - return metadata.directory.location(nodeId).rack; + @Override + public String getDatacenter(InetAddressAndPort endpoint) + { + return inferLocation(endpoint).datacenter; } } diff --git a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java index 4ff726c465..346aabe44d 100644 --- a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java +++ b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java @@ -40,14 +40,12 @@ import static org.apache.cassandra.auth.IInternodeAuthenticator.InternodeConnect public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber { private static final Logger logger = LoggerFactory.getLogger(ReconnectableSnitchHelper.class); - private final IEndpointSnitch snitch; - private final String localDc; + private final Locator locator; private final boolean preferLocal; - public ReconnectableSnitchHelper(IEndpointSnitch snitch, String localDc, boolean preferLocal) + public ReconnectableSnitchHelper(Locator locator, boolean preferLocal) { - this.snitch = snitch; - this.localDc = localDc; + this.locator = locator; this.preferLocal = preferLocal; } @@ -55,7 +53,7 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber { try { - reconnect(publicAddress, InetAddressAndPort.getByName(localAddressValue.value), snitch, localDc); + reconnect(publicAddress, InetAddressAndPort.getByName(localAddressValue.value), locator); } catch (UnknownHostException e) { @@ -64,7 +62,7 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber } @VisibleForTesting - static void reconnect(InetAddressAndPort publicAddress, InetAddressAndPort localAddress, IEndpointSnitch snitch, String localDc) + static void reconnect(InetAddressAndPort publicAddress, InetAddressAndPort localAddress, Locator locator) { final OutboundConnectionSettings settings = new OutboundConnectionSettings(publicAddress, localAddress).withDefaults(ConnectionCategory.MESSAGING); if (!settings.authenticator().authenticate(settings.to.getAddress(), settings.to.getPort(), null, OUTBOUND_PRECONNECT)) @@ -73,18 +71,13 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber return; } - if (snitch.getDatacenter(publicAddress).equals(localDc)) + if (locator.local().sameDatacenter(locator.location(publicAddress))) { MessagingService.instance().maybeReconnectWithNewIp(publicAddress, localAddress); logger.debug("Initiated reconnect to an Internal IP {} for the {}", localAddress, publicAddress); } } - public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) - { - // no-op - } - public void onJoin(InetAddressAndPort endpoint, EndpointState epState) { if (preferLocal && !Gossiper.instance.isDeadState(epState)) @@ -127,19 +120,4 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber if (preferLocal && internalIP != null) reconnect(endpoint, internalIPAndPorts != null ? internalIPAndPorts : internalIP); } - - public void onDead(InetAddressAndPort endpoint, EndpointState state) - { - // do nothing. - } - - public void onRemove(InetAddressAndPort endpoint) - { - // do nothing. - } - - public void onRestart(InetAddressAndPort endpoint, EndpointState state) - { - // do nothing. - } } diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java index ed5666a26c..751737fafb 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaLayout.java +++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java @@ -360,7 +360,7 @@ public abstract class ReplicaLayout> EndpointsForToken replicas = keyspace.getMetadata().params.replication.isLocal() ? forLocalStrategyToken(metadata, replicationStrategy, token) : forNonLocalStrategyTokenRead(metadata, keyspace.getMetadata(), token); - replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); + replicas = DatabaseDescriptor.getNodeProximity().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); replicas = replicas.filter(FailureDetector.isReplicaAlive); return new ReplicaLayout.ForTokenRead(replicationStrategy, replicas); } @@ -376,7 +376,7 @@ public abstract class ReplicaLayout> ? forLocalStrategyRange(metadata, replicationStrategy, range) : forNonLocalStategyRangeRead(metadata, keyspace.getMetadata(), range); - replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); + replicas = DatabaseDescriptor.getNodeProximity().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); replicas = replicas.filter(FailureDetector.isReplicaAlive); return new ReplicaLayout.ForRangeRead(replicationStrategy, range, replicas); } diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java index 51830ffa4b..62b7f0ca2d 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java @@ -68,6 +68,8 @@ import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.Directory; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.utils.FBUtilities; @@ -98,7 +100,7 @@ public class ReplicaPlans logger.warn("System property {} was set to {} but must be 1 or 2. Running with {}", CassandraRelevantProperties.REQUIRED_BATCHLOG_REPLICA_COUNT.getKey(), batchlogReplicaCount, REQUIRED_BATCHLOG_REPLICA_COUNT); } - public static boolean isSufficientLiveReplicasForRead(AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints liveReplicas) + public static boolean isSufficientLiveReplicasForRead(Locator locator, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints liveReplicas) { switch (consistencyLevel) { @@ -114,7 +116,7 @@ public class ReplicaPlans { int fullCount = 0; Collection dcs = ((NetworkTopologyStrategy) replicationStrategy).getDatacenters(); - for (ObjectObjectCursor entry : countPerDc(dcs, liveReplicas)) + for (ObjectObjectCursor entry : countPerDc(locator, dcs, liveReplicas)) { Replicas.ReplicaCount count = entry.value; if (!count.hasAtleast(localQuorumFor(replicationStrategy, entry.key), 0)) @@ -130,17 +132,17 @@ public class ReplicaPlans } } - static void assureSufficientLiveReplicasForRead(AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints liveReplicas) throws UnavailableException + static void assureSufficientLiveReplicasForRead(Locator locator, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints liveReplicas) throws UnavailableException { - assureSufficientLiveReplicas(replicationStrategy, consistencyLevel, liveReplicas, consistencyLevel.blockFor(replicationStrategy), 1); + assureSufficientLiveReplicas(locator, replicationStrategy, consistencyLevel, liveReplicas, consistencyLevel.blockFor(replicationStrategy), 1); } - static void assureSufficientLiveReplicasForWrite(AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints allLive, Endpoints pendingWithDown) throws UnavailableException + static void assureSufficientLiveReplicasForWrite(Locator locator, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints allLive, Endpoints pendingWithDown) throws UnavailableException { - assureSufficientLiveReplicas(replicationStrategy, consistencyLevel, allLive, consistencyLevel.blockForWrite(replicationStrategy, pendingWithDown), 0); + assureSufficientLiveReplicas(locator, replicationStrategy, consistencyLevel, allLive, consistencyLevel.blockForWrite(replicationStrategy, pendingWithDown), 0); } - static void assureSufficientLiveReplicas(AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints allLive, int blockFor, int blockForFullReplicas) throws UnavailableException + static void assureSufficientLiveReplicas(Locator locator, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, Endpoints allLive, int blockFor, int blockForFullReplicas) throws UnavailableException { switch (consistencyLevel) { @@ -172,7 +174,7 @@ public class ReplicaPlans int total = 0; int totalFull = 0; Collection dcs = ((NetworkTopologyStrategy) replicationStrategy).getDatacenters(); - for (ObjectObjectCursor entry : countPerDc(dcs, allLive)) + for (ObjectObjectCursor entry : countPerDc(locator, dcs, allLive)) { int dcBlockFor = localQuorumFor(replicationStrategy, entry.key); Replicas.ReplicaCount dcCount = entry.value; @@ -226,7 +228,7 @@ public class ReplicaPlans public static Replica findCounterLeaderReplica(ClusterMetadata metadata, String keyspaceName, DecoratedKey key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException { Keyspace keyspace = Keyspace.open(keyspaceName); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); EndpointsForToken replicas = metadata.placements.get(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get(); @@ -243,7 +245,7 @@ public class ReplicaPlans List localReplicas = new ArrayList<>(replicas.size()); for (Replica replica : replicas) - if (snitch.getDatacenter(replica).equals(localDataCenter)) + if (metadata.locator.location(replica.endpoint()).datacenter.equals(localDataCenter)) localReplicas.add(replica); if (localReplicas.isEmpty()) @@ -252,8 +254,8 @@ public class ReplicaPlans if (cl.isDatacenterLocal()) throw UnavailableException.create(cl, cl.blockFor(replicationStrategy), 0); - // No endpoint in local DC, pick the closest endpoint according to the snitch - replicas = snitch.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); + // No endpoint in local DC, pick the closest endpoint according to the configured proximity measures + replicas = proximity.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); return replicas.get(0); } @@ -296,16 +298,17 @@ public class ReplicaPlans private static ReplicaLayout.ForTokenWrite liveAndDownForBatchlogWrite(Token token, ClusterMetadata metadata, boolean isAny) { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - Multimap localEndpoints = HashMultimap.create(metadata.directory.allDatacenterRacks() - .get(snitch.getLocalDatacenter())); + Directory directory = metadata.directory; + Location local = metadata.locator.local(); + Multimap localEndpoints = HashMultimap.create(directory.allDatacenterRacks() + .get(local.datacenter)); // Replicas are picked manually: // - replicas should be alive according to the failure detector // - replicas should be in the local datacenter // - choose min(2, number of qualifying candiates above) // - allow the local node to be the only replica only if it's a single-node DC Collection chosenEndpoints = filterBatchlogEndpoints(false, - snitch.getLocalRack(), + local.rack, localEndpoints, Collections::shuffle, (r) -> FailureDetector.isEndpointAlive.test(r) && metadata.directory.peerState(r) == NodeState.JOINED, @@ -343,7 +346,7 @@ public class ReplicaPlans AbstractReplicationStrategy replicationStrategy = liveAndDown.replicationStrategy(); EndpointsForToken contacts = writeAll.select(consistencyLevel, liveAndDown, liveAndDown); - assureSufficientLiveReplicasForWrite(replicationStrategy, consistencyLevel, liveAndDown.all(), liveAndDown.pending()); + assureSufficientLiveReplicasForWrite(metadata.locator, replicationStrategy, consistencyLevel, liveAndDown.all(), liveAndDown.pending()); return new ReplicaPlan.ForWrite(systemKeyspace, replicationStrategy, consistencyLevel, @@ -506,7 +509,7 @@ public class ReplicaPlans if (racks.isEmpty()) racks.addAll(validated.keySet()); - String rack = DatabaseDescriptor.getEndpointSnitch().getRack(endpoint); + String rack = DatabaseDescriptor.getLocator().location(endpoint).rack; if (!racks.remove(rack)) continue; if (result.contains(endpoint)) @@ -523,7 +526,7 @@ public class ReplicaPlans public static List sortByProximity(Collection endpoints) { EndpointsForRange endpointsForRange = SystemReplicas.getSystemReplicas(endpoints); - return DatabaseDescriptor.getEndpointSnitch() + return DatabaseDescriptor.getNodeProximity() .sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpointsForRange) .endpointList(); } @@ -538,7 +541,7 @@ public class ReplicaPlans ReplicaLayout.ForTokenWrite live = liveAndDown.filter(isAlive); EndpointsForToken contacts = selector.select(consistencyLevel, liveAndDown, live); - assureSufficientLiveReplicasForWrite(replicationStrategy, consistencyLevel, live.all(), liveAndDown.pending()); + assureSufficientLiveReplicasForWrite(metadata.locator, replicationStrategy, consistencyLevel, live.all(), liveAndDown.pending()); return new ReplicaPlan.ForWrite(keyspace, replicationStrategy, consistencyLevel, @@ -588,7 +591,7 @@ public class ReplicaPlans AbstractReplicationStrategy replicationStrategy = liveAndDown.replicationStrategy(); EndpointsForToken contacts = selector.select(consistencyLevel, liveAndDown, live); - assureSufficientLiveReplicasForWrite(replicationStrategy, consistencyLevel, live.all(), liveAndDown.pending()); + assureSufficientLiveReplicasForWrite(metadata.locator, replicationStrategy, consistencyLevel, live.all(), liveAndDown.pending()); return new ReplicaPlan.ForWrite(keyspace, replicationStrategy, @@ -655,14 +658,15 @@ public class ReplicaPlans * soft-ensure that we reach QUORUM in all DCs we are able to, by writing to every node; * even if we don't wait for ACK, we have in both cases sent sufficient messages. */ - ObjectIntHashMap requiredPerDc = eachQuorumForWrite(liveAndDown.replicationStrategy(), liveAndDown.pending()); - addToCountPerDc(requiredPerDc, live.natural().filter(Replica::isFull), -1); - addToCountPerDc(requiredPerDc, live.pending(), -1); + Locator locator = ClusterMetadata.current().locator; + ObjectIntHashMap requiredPerDc = eachQuorumForWrite(locator, liveAndDown.replicationStrategy(), liveAndDown.pending()); + addToCountPerDc(locator, requiredPerDc, live.natural().filter(Replica::isFull), -1); + addToCountPerDc(locator, requiredPerDc, live.pending(), -1); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); for (Replica replica : filter(live.natural(), Replica::isTransient)) { - String dc = snitch.getDatacenter(replica); + + String dc = locator.location(replica.endpoint()).datacenter; if (requiredPerDc.addTo(dc, -1) >= 0) contacts.add(replica); } @@ -713,12 +717,12 @@ public class ReplicaPlans } else { - ObjectIntHashMap requiredPerDc = eachQuorumForWrite(liveAndDown.replicationStrategy(), liveAndDown.pending()); - addToCountPerDc(requiredPerDc, contacts.snapshot(), -1); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + Locator locator = ClusterMetadata.current().locator; + ObjectIntHashMap requiredPerDc = eachQuorumForWrite(locator, liveAndDown.replicationStrategy(), liveAndDown.pending()); + addToCountPerDc(locator, requiredPerDc, contacts.snapshot(), -1); for (Replica replica : filter(live.all(), r -> !contacts.contains(r))) { - String dc = snitch.getDatacenter(replica); + String dc = locator.location(replica.endpoint()).datacenter; if (requiredPerDc.addTo(dc, -1) >= 0) contacts.add(replica); } @@ -796,18 +800,16 @@ public class ReplicaPlans return indexQueryPlan != null ? IndexStatusManager.instance.filterForQuery(replicas, keyspace, indexQueryPlan, consistencyLevel) : replicas; } - private static > E contactForEachQuorumRead(NetworkTopologyStrategy replicationStrategy, E candidates) + private static > E contactForEachQuorumRead(Locator locator, NetworkTopologyStrategy replicationStrategy, E candidates) { ObjectIntHashMap perDc = eachQuorumForRead(replicationStrategy); - - final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); return candidates.filter(replica -> { - String dc = snitch.getDatacenter(replica); + String dc = locator.location(replica.endpoint()).datacenter; return perDc.addTo(dc, -1) >= 0; }); } - private static > E contactForRead(AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, boolean alwaysSpeculate, E candidates) + private static > E contactForRead(Locator locator, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, boolean alwaysSpeculate, E candidates) { /* * If we are doing an each quorum query, we have to make sure that the endpoints we select @@ -819,7 +821,7 @@ public class ReplicaPlans * TODO: this is still very inconistently managed between {LOCAL,EACH}_QUORUM and other consistency levels - should address this in a follow-up */ if (consistencyLevel == EACH_QUORUM && replicationStrategy instanceof NetworkTopologyStrategy) - return contactForEachQuorumRead((NetworkTopologyStrategy) replicationStrategy, candidates); + return contactForEachQuorumRead(locator, (NetworkTopologyStrategy) replicationStrategy, candidates); int count = consistencyLevel.blockFor(replicationStrategy) + (alwaysSpeculate ? 1 : 0); return candidates.subList(0, Math.min(count, candidates.size())); @@ -903,10 +905,10 @@ public class ReplicaPlans AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); ReplicaLayout.ForTokenRead forTokenRead = ReplicaLayout.forTokenReadLiveSorted(metadata, keyspace, replicationStrategy, token); EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, forTokenRead.natural()); - EndpointsForToken contacts = contactForRead(replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates); + EndpointsForToken contacts = contactForRead(metadata.locator, replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates); if (throwOnInsufficientLiveReplicas) - assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts); + assureSufficientLiveReplicasForRead(metadata.locator, replicationStrategy, consistencyLevel, contacts); return new ReplicaPlan.ForTokenRead(keyspace, replicationStrategy, consistencyLevel, candidates, contacts, (newClusterMetadata) -> forRead(newClusterMetadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, false), @@ -941,10 +943,10 @@ public class ReplicaPlans AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); ReplicaLayout.ForRangeRead forRangeRead = ReplicaLayout.forRangeReadLiveSorted(metadata, keyspace, replicationStrategy, range); EndpointsForRange candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, forRangeRead.natural()); - EndpointsForRange contacts = contactForRead(replicationStrategy, consistencyLevel, false, candidates); + EndpointsForRange contacts = contactForRead(metadata.locator, replicationStrategy, consistencyLevel, false, candidates); if (throwOnInsufficientLiveReplicas) - assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts); + assureSufficientLiveReplicasForRead(metadata.locator, replicationStrategy, consistencyLevel, contacts); return new ReplicaPlan.ForRangeRead(keyspace, replicationStrategy, @@ -986,7 +988,8 @@ public class ReplicaPlans /** * Take two range read plans for adjacent ranges, and check if it is OK (and worthwhile) to combine them into a single plan */ - public static ReplicaPlan.ForRangeRead maybeMerge(Keyspace keyspace, + public static ReplicaPlan.ForRangeRead maybeMerge(ClusterMetadata metadata, + Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaPlan.ForRangeRead left, ReplicaPlan.ForRangeRead right) @@ -998,16 +1001,16 @@ public class ReplicaPlans EndpointsForRange mergedCandidates = left.readCandidates().keep(right.readCandidates().endpoints()); AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); - EndpointsForRange contacts = contactForRead(replicationStrategy, consistencyLevel, false, mergedCandidates); + EndpointsForRange contacts = contactForRead(metadata.locator, replicationStrategy, consistencyLevel, false, mergedCandidates); // Estimate whether merging will be a win or not - if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(contacts, left.contacts(), right.contacts())) + if (!DatabaseDescriptor.getNodeProximity().isWorthMergingForRangeQuery(contacts, left.contacts(), right.contacts())) return null; AbstractBounds newRange = left.range().withNewRight(right.range().right); // Check if there are enough shared endpoints for the merge to be possible. - if (!isSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, mergedCandidates)) + if (!isSufficientLiveReplicasForRead(metadata.locator, replicationStrategy, consistencyLevel, mergedCandidates)) return null; int newVnodeCount = left.vnodeCount() + right.vnodeCount(); diff --git a/src/java/org/apache/cassandra/locator/Replicas.java b/src/java/org/apache/cassandra/locator/Replicas.java index c53815c563..e280e9980d 100644 --- a/src/java/org/apache/cassandra/locator/Replicas.java +++ b/src/java/org/apache/cassandra/locator/Replicas.java @@ -25,8 +25,8 @@ import java.util.function.Predicate; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectObjectHashMap; + import com.google.common.collect.Iterables; -import org.apache.cassandra.config.DatabaseDescriptor; import static com.google.common.collect.Iterables.all; @@ -88,16 +88,15 @@ public class Replicas /** * count the number of full and transient replicas, separately, for each DC */ - public static ObjectObjectHashMap countPerDc(Collection dataCenters, Iterable replicas) + public static ObjectObjectHashMap countPerDc(Locator locator, Collection dataCenters, Iterable replicas) { ObjectObjectHashMap perDc = new ObjectObjectHashMap<>(dataCenters.size()); for (String dc: dataCenters) perDc.put(dc, new ReplicaCount()); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); for (Replica replica : replicas) { - String dc = snitch.getDatacenter(replica); + String dc = locator.location(replica.endpoint()).datacenter; perDc.get(dc).increment(replica); } return perDc; @@ -106,12 +105,11 @@ public class Replicas /** * increment each of the map's DC entries for each matching replica provided */ - public static void addToCountPerDc(ObjectIntHashMap perDc, Iterable replicas, int add) + public static void addToCountPerDc(Locator locator, ObjectIntHashMap perDc, Iterable replicas, int add) { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); for (Replica replica : replicas) { - String dc = snitch.getDatacenter(replica); + String dc = locator.location(replica.endpoint()).datacenter; perDc.addTo(dc, add); } } diff --git a/src/java/org/apache/cassandra/locator/SimpleLocationProvider.java b/src/java/org/apache/cassandra/locator/SimpleLocationProvider.java new file mode 100644 index 0000000000..3217b2d16b --- /dev/null +++ b/src/java/org/apache/cassandra/locator/SimpleLocationProvider.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.tcm.membership.Location; + +public class SimpleLocationProvider implements InitialLocationProvider +{ + public static final Location LOCATION = new Location("datacenter1", "rack1"); + + @Override + public Location initialLocation() + { + return LOCATION; + } +} diff --git a/src/java/org/apache/cassandra/locator/SimpleSnitch.java b/src/java/org/apache/cassandra/locator/SimpleSnitch.java index f9bf215258..e06316fa26 100644 --- a/src/java/org/apache/cassandra/locator/SimpleSnitch.java +++ b/src/java/org/apache/cassandra/locator/SimpleSnitch.java @@ -21,34 +21,41 @@ package org.apache.cassandra.locator; * A simple endpoint snitch implementation that treats Strategy order as proximity, * allowing non-read-repaired reads to prefer a single endpoint, which improves * cache locality. + * @deprecated See CASSANDRA-19488 */ +@Deprecated(since = "CEP-21") public class SimpleSnitch extends AbstractEndpointSnitch { - public static final String DATA_CENTER_NAME = "datacenter1"; - public static final String RACK_NAME = "rack1"; + private static final NodeProximity sorter = new NoOpProximity(); + private static final SimpleLocationProvider provider = new SimpleLocationProvider(); - public String getRack(InetAddressAndPort endpoint) + @Override + public String getLocalRack() { - return RACK_NAME; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - return DATA_CENTER_NAME; + return provider.initialLocation().rack; } @Override - public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddress) + public String getLocalDatacenter() { - // Optimization to avoid walking the list - return unsortedAddress; + return provider.initialLocation().datacenter; + } + + @Override + public > C sortedByProximity(InetAddressAndPort address, C unsortedAddress) + { + return sorter.sortedByProximity(address, unsortedAddress); } @Override public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) { - // Making all endpoints equal ensures we won't change the original ordering (since - // Collections.sort is guaranteed to be stable) - return 0; + return sorter.compareEndpoints(target, r1, r2); + } + + @Override + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) + { + return sorter.isWorthMergingForRangeQuery(merged, l1, l2); } } diff --git a/src/java/org/apache/cassandra/locator/SnitchAdapter.java b/src/java/org/apache/cassandra/locator/SnitchAdapter.java new file mode 100644 index 0000000000..1a32dd6497 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/SnitchAdapter.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; + +public class SnitchAdapter implements InitialLocationProvider, NodeProximity, NodeAddressConfig +{ + public final IEndpointSnitch snitch; + + public SnitchAdapter(IEndpointSnitch snitch) + { + this.snitch = snitch; + } + + @Override + public Location initialLocation() + { + return new Location(snitch.getLocalDatacenter(), snitch.getLocalRack()); + } + + @Override + public void validate(ClusterMetadata metadata) + { + Set datacenters = metadata.directory.allDatacenterRacks().keySet(); + Set racks = new HashSet<>(); + for (String dc : datacenters) + racks.addAll(metadata.directory.datacenterRacks(dc).keySet()); + if (!snitch.validate(datacenters, racks)) + throw new ConfigurationException("Initial location provider rejected registration location, " + + "please check the system log for errors"); + } + + @Override + public > C sortedByProximity(InetAddressAndPort address, C addresses) + { + return snitch.sortedByProximity(address, addresses); + } + + @Override + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) + { + return snitch.compareEndpoints(target, r1, r2); + } + + @Override + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) + { + return snitch.isWorthMergingForRangeQuery(merged, l1, l2); + } + + @Override + public void configureAddresses() + { + snitch.configureAddresses(); + } + + @Override + public boolean preferLocalConnections() + { + return snitch.preferLocalConnections(); + } +} diff --git a/src/java/org/apache/cassandra/locator/SnitchUtils.java b/src/java/org/apache/cassandra/locator/SnitchUtils.java index 65c1f28c06..11d9bfa68f 100644 --- a/src/java/org/apache/cassandra/locator/SnitchUtils.java +++ b/src/java/org/apache/cassandra/locator/SnitchUtils.java @@ -18,6 +18,7 @@ package org.apache.cassandra.locator; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.Pair; import static java.lang.String.format; @@ -26,7 +27,13 @@ public class SnitchUtils { private SnitchUtils() {} - static Pair parseDcAndRack(String response, String dcSuffix) + public static Location parseLocation(String response, String dcSuffix) + { + Pair pair = parseDcAndRack(response, dcSuffix); + return new Location(pair.left, pair.right); + } + + public static Pair parseDcAndRack(String response, String dcSuffix) { String[] splits = response.split("/"); String az = splits[splits.length - 1]; diff --git a/src/java/org/apache/cassandra/locator/TopologyFileLocationProvider.java b/src/java/org/apache/cassandra/locator/TopologyFileLocationProvider.java new file mode 100644 index 0000000000..9ee4b696f5 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/TopologyFileLocationProvider.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.io.InputStream; +import java.net.UnknownHostException; +import java.util.Map; +import java.util.Properties; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Emulates the previous behaviour of PropertyFileSnitch; reads cassandra-topology.properties + * to identify the local datacenter and rack for one time use during node registration. + * + * Based on a properties file in the following format: + * + * 10.0.0.13=DC1:RAC2 + * 10.21.119.14=DC3:RAC2 + * 10.20.114.15=DC2:RAC2 + * default=DC1:r1 + * + * Post CEP-21, only the local rack and DC are loaded from file. Each peer in the cluster is required to register + * itself with the Cluster Metadata Service and provide its Location (Rack + DC) before joining. During upgrades, + * this is done automatically with location derived from gossip state (ultimately from system.local). + * Once registered, the Rack & DC should not be changed but currently the only safeguards against this are the + * StartupChecks which validate the snitch against system.local. + */ +public class TopologyFileLocationProvider implements InitialLocationProvider +{ + private static final Logger logger = LoggerFactory.getLogger(TopologyFileLocationProvider.class); + + public static final String PROPERTIES_FILENAME = "cassandra-topology.properties"; + // All the defaults + private static final String DEFAULT_PROPERTY = "default"; + @VisibleForTesting + public static final String DEFAULT_DC = "default"; + @VisibleForTesting + public static final String DEFAULT_RACK = "default"; + + // Used only during initialization of a new node. This provides the location it will register in cluster metadata + private final Location local; + + public TopologyFileLocationProvider() + { + local = loadConfiguration(); + } + + @Override + public Location initialLocation() + { + return local; + } + + private Location makeLocation(String value) + { + if (value == null || value.isEmpty()) + return null; + + String[] parts = value.split(":"); + if (parts.length < 2) + { + return new Location(DEFAULT_DC, DEFAULT_RACK); + } + else + { + return new Location(parts[0].trim(), parts[1].trim()); + } + } + + private Location loadConfiguration() throws ConfigurationException + { + Properties properties = new Properties(); + try (InputStream stream = getClass().getClassLoader().getResourceAsStream(PROPERTIES_FILENAME)) + { + properties.load(stream); + } + catch (Exception e) + { + throw new ConfigurationException("Unable to read " + PROPERTIES_FILENAME, e); + } + + Location local = null; + InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); + for (Map.Entry entry : properties.entrySet()) + { + String key = (String) entry.getKey(); + String value = (String) entry.getValue(); + if (DEFAULT_PROPERTY.equals(key)) + continue; + + String hostString = StringUtils.remove(key, '/'); + try + { + InetAddressAndPort host = InetAddressAndPort.getByName(hostString); + if (host.equals(broadcastAddress)) + { + local = makeLocation(value); + break; + } + } + catch (UnknownHostException e) + { + throw new ConfigurationException("Unknown host " + hostString, e); + } + + } + + // This may be null, which is ok unless config doesn't contain the location of the local node + Location defaultLocation = makeLocation(properties.getProperty(DEFAULT_PROPERTY)); + + if (local == null) + { + if (defaultLocation == null) + { + throw new ConfigurationException(String.format("Snitch definitions at %s do not define a location for " + + "this node's broadcast address %s, nor does it provides a default", + PROPERTIES_FILENAME, broadcastAddress)); + } + else + { + logger.debug("Broadcast address {} was not present in snitch config, using default location {}. " + + "This only matters on first boot, before registering with the cluster metadata service", + broadcastAddress, defaultLocation); + local = defaultLocation; + } + } + + logger.debug("Loaded location {} for broadcast address {} from property file. " + + "This only matters on first boot, before registering with the cluster metadata service", + local, broadcastAddress); + + return local; + } + +} diff --git a/src/java/org/apache/cassandra/metrics/MessagingMetrics.java b/src/java/org/apache/cassandra/metrics/MessagingMetrics.java index 6151f57ee4..8f122d6dd7 100644 --- a/src/java/org/apache/cassandra/metrics/MessagingMetrics.java +++ b/src/java/org/apache/cassandra/metrics/MessagingMetrics.java @@ -114,7 +114,7 @@ public class MessagingMetrics implements InboundMessageHandlers.GlobalMetricCall public DCLatencyRecorder internodeLatencyRecorder(InetAddressAndPort from) { - String dcName = DatabaseDescriptor.getEndpointSnitch().getDatacenter(from); + String dcName = DatabaseDescriptor.getLocator().location(from).datacenter; DCLatencyRecorder dcUpdater = dcLatency.get(dcName); if (dcUpdater == null) dcUpdater = dcLatency.computeIfAbsent(dcName, k -> new DCLatencyRecorder(Metrics.timer(factory.createMetricName(dcName + "-Latency")), allLatency)); diff --git a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java index ccc74f0aba..23ac75e717 100644 --- a/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java +++ b/src/java/org/apache/cassandra/net/OutboundConnectionSettings.java @@ -28,11 +28,10 @@ import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions; import org.apache.cassandra.db.SystemKeyspace; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.utils.FBUtilities; -import static org.apache.cassandra.config.DatabaseDescriptor.getEndpointSnitch; import static org.apache.cassandra.net.MessagingService.instance; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; @@ -420,7 +419,7 @@ public class OutboundConnectionSettings if (tcpNoDelay != null) return tcpNoDelay; - if (DatabaseDescriptor.isClientOrToolInitialized() || isInLocalDC(getEndpointSnitch(), getBroadcastAddressAndPort(), to)) + if (DatabaseDescriptor.isClientOrToolInitialized() || isInLocalDC(getBroadcastAddressAndPort(), to)) return INTRADC_TCP_NODELAY; return DatabaseDescriptor.getInterDCTcpNoDelay(); @@ -459,7 +458,7 @@ public class OutboundConnectionSettings if (category.isStreaming()) return Framing.UNPROTECTED; - return shouldCompressConnection(getEndpointSnitch(), getBroadcastAddressAndPort(), to) + return shouldCompressConnection(getBroadcastAddressAndPort(), to) ? Framing.LZ4 : Framing.CRC; } @@ -479,10 +478,11 @@ public class OutboundConnectionSettings from(), socketFactory(), callbacks(), debug(), endpointToVersion()); } - private static boolean isInLocalDC(IEndpointSnitch snitch, InetAddressAndPort localHost, InetAddressAndPort remoteHost) + private static boolean isInLocalDC(InetAddressAndPort localHost, InetAddressAndPort remoteHost) { - String remoteDC = snitch.getDatacenter(remoteHost); - String localDC = snitch.getDatacenter(localHost); + Locator locator = DatabaseDescriptor.getLocator(); + String remoteDC = locator.location(remoteHost).datacenter; + String localDC = locator.location(localHost).datacenter; return remoteDC != null && remoteDC.equals(localDC); } @@ -494,10 +494,10 @@ public class OutboundConnectionSettings } @VisibleForTesting - static boolean shouldCompressConnection(IEndpointSnitch snitch, InetAddressAndPort localHost, InetAddressAndPort remoteHost) + static boolean shouldCompressConnection(InetAddressAndPort localHost, InetAddressAndPort remoteHost) { return (DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.all) - || ((DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.dc) && !isInLocalDC(snitch, localHost, remoteHost)); + || ((DatabaseDescriptor.internodeCompression() == Config.InternodeCompression.dc) && !isInLocalDC(localHost, remoteHost)); } } diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 63b7b96ec5..7b60df9416 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -488,7 +488,7 @@ public class RepairJob extends AsyncFuture implements Runnable private String getDC(InetAddressAndPort address) { - return ctx.snitch().getDatacenter(address); + return ctx.locator().location(address).datacenter; } /** @@ -573,7 +573,7 @@ public class RepairJob extends AsyncFuture implements Runnable Map> requestsByDatacenter = new HashMap<>(); for (InetAddressAndPort endpoint : endpoints) { - String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint); + String dc = DatabaseDescriptor.getLocator().location(endpoint).datacenter; Queue queue = requestsByDatacenter.computeIfAbsent(dc, k -> new LinkedList<>()); queue.add(endpoint); } diff --git a/src/java/org/apache/cassandra/repair/SharedContext.java b/src/java/org/apache/cassandra/repair/SharedContext.java index 6c13ae99c9..54ec421457 100644 --- a/src/java/org/apache/cassandra/repair/SharedContext.java +++ b/src/java/org/apache/cassandra/repair/SharedContext.java @@ -32,8 +32,8 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.gms.IGossiper; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.ActiveRepairService; @@ -73,7 +73,7 @@ public interface SharedContext }; } IFailureDetector failureDetector(); - IEndpointSnitch snitch(); + Locator locator(); IGossiper gossiper(); ICompactionManager compactionManager(); ActiveRepairService repair(); @@ -147,9 +147,9 @@ public interface SharedContext } @Override - public IEndpointSnitch snitch() + public Locator locator() { - return DatabaseDescriptor.getEndpointSnitch(); + return DatabaseDescriptor.getLocator(); } @Override @@ -270,9 +270,9 @@ public interface SharedContext } @Override - public IEndpointSnitch snitch() + public Locator locator() { - return delegate().snitch(); + return delegate().locator(); } @Override diff --git a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java index 4607a0fcf5..37af87f3e4 100644 --- a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java +++ b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java @@ -20,6 +20,7 @@ package org.apache.cassandra.schema; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.Set; import java.util.function.Supplier; @@ -224,4 +225,9 @@ public final class DistributedMetadataLogKeyspace { return KeyspaceMetadata.create(SchemaConstants.METADATA_KEYSPACE_NAME, new KeyspaceParams(true, ReplicationParams.simpleMeta(1, knownDatacenters)), Tables.of(Log)); } + + public static KeyspaceMetadata initialMetadata(String datacenter) + { + return initialMetadata(Collections.singleton(datacenter)); + } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/schema/DistributedSchema.java b/src/java/org/apache/cassandra/schema/DistributedSchema.java index ec0c65eb3a..e4eead15b5 100644 --- a/src/java/org/apache/cassandra/schema/DistributedSchema.java +++ b/src/java/org/apache/cassandra/schema/DistributedSchema.java @@ -62,13 +62,20 @@ public class DistributedSchema implements MetadataValue public static DistributedSchema first(Set knownDatacenters) { + // During upgrades from pre-5.1 versions, the replication params of the system_cluster_metadata + // keyspace using one of the existing DCs. This is so that this keyspace does not cause issues + // for tooling, clients or control plane systems which may inspect schema and have specific + // expectations about DC layout. This keyspace is unused until the CMS is initialized. + // For new clusters which start out on 5.1 or later, this is not necessary to the initial + // replication params use a empty string for the placeholder DC name. + + // During CMS initialization, the replication of this keyspace will be set for real using + // the DC of the first node to become a CMS member. This happens in the PreInitialize + // transformation when executed on the first CMS member. + if (knownDatacenters.isEmpty()) - { - if (DatabaseDescriptor.getLocalDataCenter() != null) - knownDatacenters = Collections.singleton(DatabaseDescriptor.getLocalDataCenter()); - else - knownDatacenters = Collections.singleton("DC1"); - } + knownDatacenters = Collections.singleton(""); + return new DistributedSchema(Keyspaces.of(DistributedMetadataLogKeyspace.initialMetadata(knownDatacenters)), Epoch.FIRST); } @@ -216,7 +223,7 @@ public class DistributedSchema implements MetadataValue }); // Avoid system table side effects during initialization - if (epoch.isEqualOrAfter(Epoch.FIRST)) + if (epoch.isEqualOrAfter(Epoch.FIRST) && !DatabaseDescriptor.isClientOrToolInitialized()) { Collection mutations = SchemaKeyspace.convertSchemaDiffToMutations(ksDiff, FBUtilities.timestampMicros()); SchemaKeyspace.applyChanges(mutations); diff --git a/src/java/org/apache/cassandra/schema/ReplicationParams.java b/src/java/org/apache/cassandra/schema/ReplicationParams.java index cf445a4b78..277c959146 100644 --- a/src/java/org/apache/cassandra/schema/ReplicationParams.java +++ b/src/java/org/apache/cassandra/schema/ReplicationParams.java @@ -18,6 +18,7 @@ package org.apache.cassandra.schema; import java.io.IOException; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -121,6 +122,11 @@ public final class ReplicationParams return new ReplicationParams(SimpleStrategy.class, ImmutableMap.of("replication_factor", replicationFactor)); } + public static ReplicationParams simpleMeta(int replicationFactor, String datacenter) + { + return simpleMeta(replicationFactor, Collections.singleton(datacenter)); + } + public static ReplicationParams simpleMeta(int replicationFactor, Set knownDatacenters) { if (replicationFactor <= 0) diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 360c26bdcc..df73f14b40 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -65,6 +65,11 @@ import org.apache.cassandra.exceptions.StartupException; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; +import org.apache.cassandra.tcm.CMSOperations; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.RegistrationStatus; +import org.apache.cassandra.tcm.Startup; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.DefaultNameFactory; import org.apache.cassandra.net.StartupClusterConnectivityChecker; @@ -74,11 +79,8 @@ import org.apache.cassandra.security.ThreadAwareSecurityManager; import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.StreamManager; -import org.apache.cassandra.tcm.CMSOperations; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.MultiStepOperation; -import org.apache.cassandra.tcm.Startup; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JMXServerUtils; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -265,6 +267,8 @@ public class CassandraDaemon Startup.initialize(DatabaseDescriptor.getSeeds()); disableAutoCompaction(Schema.instance.distributedKeyspaces().names()); CMSOperations.initJmx(); + if (ClusterMetadata.current().myNodeId() != null) + RegistrationStatus.instance.onRegistration(); } catch (InterruptedException | ExecutionException | IOException e) { @@ -369,9 +373,13 @@ public class CassandraDaemon exitOrFail(1, "Fatal configuration error", e); } + // The local rack may have been changed at some point, which will now be reflected in cluster metadata. Update + // the system.local table just in case the actual value doesn't match what the configured location provided + // reported when the earlier call to SystemKeyspace::persistLocalMetadata was made prior to initialising cluster + // metadata. + SystemKeyspace.updateRack(ClusterMetadata.current().locator.local().rack); ScheduledExecutors.optionalTasks.execute(() -> ClusterMetadataService.instance().processor().fetchLogAndWait()); - // TODO: (TM/alexp), this can be made time-dependent // Because we are writing to the system_distributed keyspace, this should happen after that is created, which // happens in StorageService.instance.initServer() Runnable viewRebuild = () -> { @@ -383,11 +391,6 @@ public class CassandraDaemon }; ScheduledExecutors.optionalTasks.schedule(viewRebuild, StorageService.RING_DELAY_MILLIS, TimeUnit.MILLISECONDS); - - // TODO: (TM/alexp), we do not need to wait for gossip settlement anymore -// if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress())) -// Gossiper.waitToSettle(); - StorageService.instance.doAuthSetup(); // re-enable auto-compaction after replay, so correct disk boundaries are used @@ -637,8 +640,9 @@ public class CassandraDaemon { StartupClusterConnectivityChecker connectivityChecker = StartupClusterConnectivityChecker.create(DatabaseDescriptor.getBlockForPeersTimeoutInSeconds(), DatabaseDescriptor.getBlockForPeersInRemoteDatacenters()); + Locator locator = DatabaseDescriptor.getLocator(); Set peers = new HashSet<>(ClusterMetadata.current().directory.allJoinedEndpoints()); - connectivityChecker.execute(peers, DatabaseDescriptor.getEndpointSnitch()::getDatacenter); + connectivityChecker.execute(peers, ep -> locator.location(ep).datacenter); // check to see if transports may start else return without starting. This is needed when in survey mode or // when bootstrap has not completed. diff --git a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java index e4b208b582..d035286d7d 100644 --- a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java @@ -24,7 +24,7 @@ import java.util.function.Supplier; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Mutation; -import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; @@ -38,7 +38,7 @@ import org.apache.cassandra.transport.Dispatcher; */ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponseHandler { - private static final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + private static final Locator locator = DatabaseDescriptor.getLocator(); private final Map responses = new HashMap(); private final AtomicInteger acks = new AtomicInteger(0); @@ -64,14 +64,14 @@ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponse } else { - responses.put(DatabaseDescriptor.getLocalDataCenter(), new AtomicInteger(ConsistencyLevel.quorumFor(replicaPlan.replicationStrategy()))); + responses.put(locator.local().datacenter, new AtomicInteger(ConsistencyLevel.quorumFor(replicaPlan.replicationStrategy()))); } // During bootstrap, we have to include the pending endpoints or we may fail the consistency level // guarantees (see #833) for (Replica pending : replicaPlan.pending()) { - responses.get(snitch.getDatacenter(pending)).incrementAndGet(); + responses.get(locator.location(pending.endpoint()).datacenter).incrementAndGet(); } } @@ -80,8 +80,8 @@ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponse try { String dataCenter = message == null - ? DatabaseDescriptor.getLocalDataCenter() - : snitch.getDatacenter(message.from()); + ? locator.local().datacenter + : locator.location(message.from()).datacenter; responses.get(dataCenter).getAndDecrement(); acks.incrementAndGet(); diff --git a/src/java/org/apache/cassandra/service/Rebuild.java b/src/java/org/apache/cassandra/service/Rebuild.java index f089aded13..421595bab6 100644 --- a/src/java/org/apache/cassandra/service/Rebuild.java +++ b/src/java/org/apache/cassandra/service/Rebuild.java @@ -104,17 +104,17 @@ public class Rebuild RangeStreamer streamer = new RangeStreamer(metadata, StreamOperation.REBUILD, false, // no strict consistency when rebuilding - DatabaseDescriptor.getEndpointSnitch(), + DatabaseDescriptor.getNodeProximity(), StorageService.instance.streamStateStore, false, DatabaseDescriptor.getStreamingConnectionsPerHost(), rebuildMovements, null); if (sourceDc != null) - streamer.addSourceFilter(new RangeStreamer.SingleDatacenterFilter(DatabaseDescriptor.getEndpointSnitch(), sourceDc)); + streamer.addSourceFilter(new RangeStreamer.SingleDatacenterFilter(metadata.locator, sourceDc)); if (excludeLocalDatacenterNodes) - streamer.addSourceFilter(new RangeStreamer.ExcludeLocalDatacenterFilter(DatabaseDescriptor.getEndpointSnitch())); + streamer.addSourceFilter(new RangeStreamer.ExcludeLocalDatacenterFilter(metadata.locator)); if (keyspace == null) { diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java index a16864cc2a..4ec3019465 100644 --- a/src/java/org/apache/cassandra/service/StartupChecks.java +++ b/src/java/org/apache/cassandra/service/StartupChecks.java @@ -144,8 +144,6 @@ public class StartupChecks checkDataDirs, checkSSTablesFormat, checkSystemKeyspaceState, - checkDatacenter, - checkRack, checkLegacyAuthTables, new DataResurrectionCheck()); @@ -738,46 +736,6 @@ public class StartupChecks } }; - public static final StartupCheck checkDatacenter = new StartupCheck() - { - @Override - public void execute(StartupChecksOptions options) throws StartupException - { - String storedDc = SystemKeyspace.getDatacenter(); - if (storedDc != null) - { - String currentDc = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); - if (!storedDc.equals(currentDc)) - { - String formatMessage = "Cannot start node if snitch's data center (%s) differs from previous data center (%s). " + - "Please fix the snitch configuration, decommission and rebootstrap this node"; - - throw new StartupException(StartupException.ERR_WRONG_CONFIG, String.format(formatMessage, currentDc, storedDc)); - } - } - } - }; - - public static final StartupCheck checkRack = new StartupCheck() - { - @Override - public void execute(StartupChecksOptions options) throws StartupException - { - String storedRack = SystemKeyspace.getRack(); - if (storedRack != null) - { - String currentRack = DatabaseDescriptor.getEndpointSnitch().getLocalRack(); - if (!storedRack.equals(currentRack)) - { - String formatMessage = "Cannot start node if snitch's rack (%s) differs from previous rack (%s). " + - "Please fix the snitch configuration, decommission and rebootstrap this node"; - - throw new StartupException(StartupException.ERR_WRONG_CONFIG, String.format(formatMessage, currentRack, storedRack)); - } - } - } - }; - public static final StartupCheck checkLegacyAuthTables = new StartupCheck() { @Override diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index e62c1bb3bd..cd03fae552 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -877,7 +877,7 @@ public class StorageProxy implements StorageProxyMBean throws UnavailableException, OverloadedException, WriteTimeoutException, WriteFailureException { Tracing.trace("Determining replicas for mutation"); - final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); + final String localDataCenter = DatabaseDescriptor.getLocator().local().datacenter; List> responseHandlers = new ArrayList<>(mutations.size()); WriteType plainWriteType = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH; @@ -1011,7 +1011,7 @@ public class StorageProxy implements StorageProxyMBean throws UnavailableException, OverloadedException, WriteTimeoutException { Tracing.trace("Determining replicas for mutation"); - final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); + final String localDataCenter = DatabaseDescriptor.getLocator().local().datacenter; long startTime = nanoTime(); @@ -1350,7 +1350,7 @@ public class StorageProxy implements StorageProxyMBean private static void syncWriteBatchedMutations(List wrappers, Stage stage, Dispatcher.RequestTime requestTime) throws WriteTimeoutException, OverloadedException { - String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(); + String localDataCenter = DatabaseDescriptor.getLocator().local().datacenter; for (WriteResponseHandlerWrapper wrapper : wrappers) { @@ -1528,7 +1528,7 @@ public class StorageProxy implements StorageProxyMBean Collections.singletonList(MessageFlag.CALL_BACK_ON_FAILURE)); } - String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(destination); + String dc = DatabaseDescriptor.getLocator().location(destination.endpoint()).datacenter; // direct writes to local DC or old Cassandra versions // (1.1 knows how to forward old-style String message IDs; updated to int in 2.0) @@ -2403,7 +2403,7 @@ public class StorageProxy implements StorageProxyMBean Set disabledDCs = DatabaseDescriptor.hintedHandoffDisabledDCs(); if (!disabledDCs.isEmpty()) { - final String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica); + final String dc = DatabaseDescriptor.getLocator().location(replica.endpoint()).datacenter; if (disabledDCs.contains(dc)) { Tracing.trace("Not hinting {} since its data center {} has been disabled {}", replica, dc, disabledDCs); diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 28d9d124e9..12981f724a 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -47,6 +47,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -137,7 +138,6 @@ import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.EndpointsByRange; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.locator.MetaStrategy; @@ -145,6 +145,8 @@ import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.RangesByEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.Replicas; +import org.apache.cassandra.locator.NodeProximity; +import org.apache.cassandra.locator.SnitchAdapter; import org.apache.cassandra.locator.SystemReplicas; import org.apache.cassandra.metrics.Sampler; import org.apache.cassandra.metrics.SamplingManager; @@ -178,6 +180,7 @@ import org.apache.cassandra.streaming.StreamState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.MultiStepOperation; +import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.compatibility.GossipHelper; import org.apache.cassandra.tcm.compatibility.TokenRingUtils; @@ -821,6 +824,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE self = Register.maybeRegister(); } + RegistrationStatus.instance.onRegistration(); Startup.maybeExecuteStartupTransformation(self); try @@ -3429,8 +3433,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } else { - // stream to the closest peer as chosen by the snitch - candidates = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(getBroadcastAddressAndPort(), candidates); + // stream to the closest peer as chosen by the configured proximity measures + candidates = DatabaseDescriptor.getNodeProximity().sortedByProximity(getBroadcastAddressAndPort(), candidates); InetAddressAndPort hintsDestinationHost = candidates.get(0).endpoint(); return ClusterMetadata.current().directory.peerId(hintsDestinationHost).toUUID(); } @@ -4144,12 +4148,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public void setDynamicUpdateInterval(int dynamicUpdateInterval) { - if (DatabaseDescriptor.getEndpointSnitch() instanceof DynamicEndpointSnitch) + if (DatabaseDescriptor.getNodeProximity() instanceof DynamicEndpointSnitch) { try { - updateSnitch(null, true, dynamicUpdateInterval, null, null); + updateProximityInternal(null, true, dynamicUpdateInterval, null, null); } catch (ClassNotFoundException e) { @@ -4164,6 +4168,18 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } public void updateSnitch(String epSnitchClassName, Boolean dynamic, Integer dynamicUpdateInterval, Integer dynamicResetInterval, Double dynamicBadnessThreshold) throws ClassNotFoundException + { + Supplier factory = () -> new SnitchAdapter(DatabaseDescriptor.createEndpointSnitch(epSnitchClassName)); + updateProximityInternal(factory, dynamic, dynamicUpdateInterval, dynamicResetInterval, dynamicBadnessThreshold); + } + + public void updateNodeProximity(String npsClassName, Boolean dynamic, Integer dynamicUpdateInterval, Integer dynamicResetInterval, Double dynamicBadnessThreshold) throws ClassNotFoundException + { + Supplier factory = () -> DatabaseDescriptor.createProximityImpl(npsClassName); + updateProximityInternal(factory, dynamic, dynamicUpdateInterval, dynamicResetInterval, dynamicBadnessThreshold); + } + + private void updateProximityInternal(Supplier implSupplier, Boolean dynamic, Integer dynamicUpdateInterval, Integer dynamicResetInterval, Double dynamicBadnessThreshold) throws ClassNotFoundException { // apply dynamic snitch configuration if (dynamicUpdateInterval != null) @@ -4173,51 +4189,53 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (dynamicBadnessThreshold != null) DatabaseDescriptor.setDynamicBadnessThreshold(dynamicBadnessThreshold); - IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch(); + NodeProximity oldProximity = DatabaseDescriptor.getNodeProximity(); // new snitch registers mbean during construction - if(epSnitchClassName != null) + if(implSupplier != null) { // need to unregister the mbean _before_ the new dynamic snitch is instantiated (and implicitly initialized // and its mbean registered) - if (oldSnitch instanceof DynamicEndpointSnitch) - ((DynamicEndpointSnitch)oldSnitch).close(); + if (oldProximity instanceof DynamicEndpointSnitch) + ((DynamicEndpointSnitch)oldProximity).close(); - IEndpointSnitch newSnitch; + NodeProximity newProximity; try { - newSnitch = DatabaseDescriptor.createEndpointSnitch(dynamic != null && dynamic, epSnitchClassName); + newProximity = implSupplier.get(); + if (dynamic != null && dynamic) + newProximity = new DynamicEndpointSnitch(newProximity); } catch (ConfigurationException e) { throw new ClassNotFoundException(e.getMessage()); } - if (newSnitch instanceof DynamicEndpointSnitch) + if (newProximity instanceof DynamicEndpointSnitch) { logger.info("Created new dynamic snitch {} with update-interval={}, reset-interval={}, badness-threshold={}", - ((DynamicEndpointSnitch)newSnitch).subsnitch.getClass().getName(), DatabaseDescriptor.getDynamicUpdateInterval(), + ((DynamicEndpointSnitch)newProximity).delegate.getClass().getName(), DatabaseDescriptor.getDynamicUpdateInterval(), DatabaseDescriptor.getDynamicResetInterval(), DatabaseDescriptor.getDynamicBadnessThreshold()); } else { - logger.info("Created new non-dynamic snitch {}", newSnitch.getClass().getName()); + logger.info("Created new non-dynamic snitch {}", newProximity.getClass().getName()); } - // point snitch references to the new instance - DatabaseDescriptor.setEndpointSnitch(newSnitch); + // point reference to the new instance + DatabaseDescriptor.setNodeProximity(newProximity); } else { - if (oldSnitch instanceof DynamicEndpointSnitch) + if (oldProximity instanceof DynamicEndpointSnitch) { logger.info("Applying config change to dynamic snitch {} with update-interval={}, reset-interval={}, badness-threshold={}", - ((DynamicEndpointSnitch)oldSnitch).subsnitch.getClass().getName(), DatabaseDescriptor.getDynamicUpdateInterval(), + ((DynamicEndpointSnitch)oldProximity).delegate.getClass().getName(), DatabaseDescriptor.getDynamicUpdateInterval(), DatabaseDescriptor.getDynamicResetInterval(), DatabaseDescriptor.getDynamicBadnessThreshold()); - DynamicEndpointSnitch snitch = (DynamicEndpointSnitch)oldSnitch; - snitch.applyConfigChanges(); + DynamicEndpointSnitch proximity = (DynamicEndpointSnitch)oldProximity; + proximity.applyConfigChanges(); } } } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 46ecc0e05a..3e556db04a 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -652,15 +652,35 @@ public interface StorageServiceMBean extends NotificationEmitter * * The parameters {@code dynamicUpdateInterval}, {@code dynamicResetInterval} and {@code dynamicBadnessThreshold} * can be specified individually to update the parameters of the dynamic snitch during runtime. - * * @param epSnitchClassName the canonical path name for a class implementing IEndpointSnitch * @param dynamic boolean that decides whether dynamicsnitch is used or not - only valid, if {@code epSnitchClassName} is specified * @param dynamicUpdateInterval integer, in ms (defaults to the value configured in cassandra.yaml, which defaults to 100) * @param dynamicResetInterval integer, in ms (defaults to the value configured in cassandra.yaml, which defaults to 600,000) * @param dynamicBadnessThreshold double, (defaults to the value configured in cassandra.yaml, which defaults to 0.0) + * @deprecated See CASSANDRA-19488 */ + @Deprecated(since = "5.1") public void updateSnitch(String epSnitchClassName, Boolean dynamic, Integer dynamicUpdateInterval, Integer dynamicResetInterval, Double dynamicBadnessThreshold) throws ClassNotFoundException; + /** + * Change NodeProximity class and dynamic-ness (and dynamic attributes) at runtime. + * This method supercedes the deprecated updateSnitch method. + * + * This method is used to change the proximity implementation and/or dynamic proximity parameters. + * If {@code proximityClassName} is specified, it will configure a new instance and make it a + * 'dynamic snitch' if {@code dynamic} is specified and {@code true}. + * + * The parameters {@code dynamicUpdateInterval}, {@code dynamicResetInterval} and {@code dynamicBadnessThreshold} + * can be specified individually to update the parameters of the dynamic snitch during runtime. + * + * @param proximityClassName the canonical path name for a class implementing NodeProximity + * @param dynamic boolean that decides whether dynamicsnitch is used or not - only valid, if {@code epSnitchClassName} is specified + * @param dynamicUpdateInterval integer, in ms (defaults to the value configured in cassandra.yaml, which defaults to 100) + * @param dynamicResetInterval integer, in ms (defaults to the value configured in cassandra.yaml, which defaults to 600,000) + * @param dynamicBadnessThreshold double, (defaults to the value configured in cassandra.yaml, which defaults to 0.0) + */ + public void updateNodeProximity(String proximityClassName, Boolean dynamic, Integer dynamicUpdateInterval, Integer dynamicResetInterval, Double dynamicBadnessThreshold) throws ClassNotFoundException; + /* Update dynamic_snitch_update_interval in ms */ diff --git a/src/java/org/apache/cassandra/service/TokenRange.java b/src/java/org/apache/cassandra/service/TokenRange.java index 37971f5e4f..27d1485094 100644 --- a/src/java/org/apache/cassandra/service/TokenRange.java +++ b/src/java/org/apache/cassandra/service/TokenRange.java @@ -22,8 +22,9 @@ import java.util.*; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; +import org.apache.cassandra.tcm.membership.Location; /** * Holds token range informations for the sake of {@link StorageService#describeRing}. @@ -57,12 +58,15 @@ public class TokenRange public static TokenRange create(Token.TokenFactory tokenFactory, Range range, List endpoints, boolean withPorts) { List details = new ArrayList<>(endpoints.size()); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + Locator locator = DatabaseDescriptor.getLocator(); for (InetAddressAndPort ep : endpoints) + { + Location location = locator.location(ep); details.add(new EndpointDetails(ep, StorageService.instance.getNativeaddress(ep, withPorts), - snitch.getDatacenter(ep), - snitch.getRack(ep))); + location.datacenter, + location.rack)); + } return new TokenRange(tokenFactory, range, details); } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java index 943b04c30c..b5ce86794d 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java @@ -33,6 +33,7 @@ import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; @@ -214,7 +215,8 @@ public class PaxosCommit> ex private static boolean isInLocalDc(InetAddressAndPort destination) { - return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(destination)); + Locator locator = DatabaseDescriptor.getLocator(); + return locator.local().sameDatacenter(locator.location(destination)); } /** diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java index 0e3a732185..016086cbb1 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java @@ -550,12 +550,13 @@ public class PaxosRepair extends AbstractPaxosRepair ReplicationParams replication = keyspace.getMetadata().params.replication; // Special case meta keyspace as it uses a custom partitioner/tokens, but the paxos table and repairs // are based on the system partitioner + ClusterMetadata metadata = ClusterMetadata.current(); Collection allEndpoints = replication.isMeta() - ? ClusterMetadata.current().fullCMSMembers() - : ClusterMetadata.current().placements.get(replication).reads.forRange(range).endpoints(); + ? metadata.fullCMSMembers() + : metadata.placements.get(replication).reads.forRange(range).endpoints(); return hasSufficientLiveNodesForTopologyChange(allEndpoints, liveEndpoints, - DatabaseDescriptor.getEndpointSnitch()::getDatacenter, + ep -> metadata.locator.location(ep).datacenter, DatabaseDescriptor.paxoTopologyRepairNoDcChecks(), DatabaseDescriptor.paxoTopologyRepairStrictEachQuorum()); } diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java index 4d5c92280b..ca25d1a3fd 100644 --- a/src/java/org/apache/cassandra/service/reads/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java @@ -258,7 +258,7 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< private void assertWaitingFor(InetAddressAndPort from) { assert !replicaPlan().consistencyLevel().isDatacenterLocal() - || DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(from)) + || DatabaseDescriptor.getLocator().local().sameDatacenter(DatabaseDescriptor.getLocator().location(from)) : "Received read response from unexpected replica: " + from; } } diff --git a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java index f2dacd0ccf..20e9562f93 100644 --- a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java +++ b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java @@ -27,6 +27,7 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.AbstractIterator; class ReplicaPlanMerger extends AbstractIterator @@ -49,6 +50,7 @@ class ReplicaPlanMerger extends AbstractIterator return endOfData(); ReplicaPlan.ForRangeRead current = ranges.next(); + ClusterMetadata metadata = ClusterMetadata.current(); // getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take // the replication factor into account. If the intersection of live endpoints for 2 consecutive ranges @@ -64,7 +66,7 @@ class ReplicaPlanMerger extends AbstractIterator break; ReplicaPlan.ForRangeRead next = ranges.peek(); - ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(keyspace, consistency, current, next); + ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(metadata, keyspace, consistency, current, next); if (merged == null) break; diff --git a/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java b/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java index 9415f89285..1bccf29044 100644 --- a/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java +++ b/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java @@ -57,7 +57,9 @@ public class DataMovementVerbHandler implements IVerbHandler assert local.isSelf(); boolean transientAdded = false; boolean fullAdded = false; - for (Replica remote : DatabaseDescriptor.getEndpointSnitch().sortedByProximity(local.endpoint(), endpoints).filter(ep -> FailureDetector.instance.isAlive(ep.endpoint()))) + for (Replica remote : DatabaseDescriptor.getNodeProximity() + .sortedByProximity(local.endpoint(), endpoints) + .filter(ep -> FailureDetector.instance.isAlive(ep.endpoint()))) { assert !remote.isSelf(); if (remote.isFull() && !fullAdded) diff --git a/src/java/org/apache/cassandra/streaming/StreamManager.java b/src/java/org/apache/cassandra/streaming/StreamManager.java index fec8b2de21..6bfecf8aa8 100644 --- a/src/java/org/apache/cassandra/streaming/StreamManager.java +++ b/src/java/org/apache/cassandra/streaming/StreamManager.java @@ -114,9 +114,10 @@ public class StreamManager implements StreamManagerMBean this.interDCLimiter = interDCLimiter; this.throughput = throughput; this.interDCThroughput = interDCThroughput; - if (DatabaseDescriptor.getLocalDataCenter() != null && DatabaseDescriptor.getEndpointSnitch() != null) - isLocalDC = DatabaseDescriptor.getLocalDataCenter().equals( - DatabaseDescriptor.getEndpointSnitch().getDatacenter(peer)); + if (DatabaseDescriptor.getLocalDataCenter() != null && DatabaseDescriptor.getLocator() != null) + isLocalDC = DatabaseDescriptor.getLocator() + .local() + .sameDatacenter(DatabaseDescriptor.getLocator().location(peer)); else isLocalDC = true; } diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index 9862e51ecb..e2b50483ce 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -46,6 +46,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.CMSIdentifierMismatchException; @@ -96,6 +97,9 @@ public class ClusterMetadata public final InProgressSequences inProgressSequences; public final ImmutableMap, ExtensionValue> extensions; + // This isn't serialized as part of ClusterMetadata it's really just a view over the Directory. + public final Locator locator; + // These fields are lazy but only for the test purposes, since their computation requires initialization of the log ks private EndpointsForRange fullCMSReplicas; private Set fullCMSEndpoints; @@ -174,6 +178,7 @@ public class ClusterMetadata this.lockedRanges = lockedRanges; this.inProgressSequences = inProgressSequences; this.extensions = ImmutableMap.copyOf(extensions); + this.locator = Locator.usingDirectory(directory); } public Set fullCMSMembers() diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java index 845b6741f9..555cdb7f93 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java @@ -98,6 +98,9 @@ public class ClusterMetadataService throw new IllegalStateException(String.format("Cluster metadata is already initialized to %s.", instance), trace); instance = newInstance; + RegistrationStatus.instance.onInitialized(); + if (newInstance.metadata().myNodeId() != null) + RegistrationStatus.instance.onRegistration(); trace = new RuntimeException("Previously initialized trace"); } @@ -105,6 +108,7 @@ public class ClusterMetadataService public static ClusterMetadataService unsetInstance() { ClusterMetadataService tmp = instance(); + RegistrationStatus.instance.resetState(); instance = null; return tmp; } @@ -245,7 +249,8 @@ public class ClusterMetadataService { if (instance != null) return; - ClusterMetadata emptyFromSystemTables = emptyWithSchemaFromSystemTables(Collections.singleton("DC1")) + String localDC = DatabaseDescriptor.getLocalDataCenter(); + ClusterMetadata emptyFromSystemTables = emptyWithSchemaFromSystemTables(Collections.singleton(localDC)) .forceEpoch(Epoch.EMPTY); LocalLog.LogSpec logSpec = LocalLog.logSpec() @@ -274,7 +279,7 @@ public class ClusterMetadataService new PeerLogFetcher(log)); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.bootstrap(FBUtilities.getBroadcastAddressAndPort(), localDC); ClusterMetadataService.setInstance(cms); } @@ -292,6 +297,7 @@ public class ClusterMetadataService if (instance != null) return; + ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema)); } diff --git a/src/java/org/apache/cassandra/tcm/RegistrationStatus.java b/src/java/org/apache/cassandra/tcm/RegistrationStatus.java new file mode 100644 index 0000000000..736a026783 --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/RegistrationStatus.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm; + +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.annotations.VisibleForTesting; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.net.MessagingService; + +public class RegistrationStatus +{ + public enum State { INITIAL, UNREGISTERED, REGISTERED } + + private static final Logger logger = LoggerFactory.getLogger(RegistrationStatus.class); + public static final RegistrationStatus instance = new RegistrationStatus(); + private final AtomicReference state = new AtomicReference<>(State.INITIAL); + + public RegistrationStatus.State getCurrent() + { + return state.get(); + } + + @VisibleForTesting + public void resetState() + { + state.set(State.INITIAL); + } + + public void onInitialized() + { + logger.info("Node is initialized, moving to UNREGISTERED state"); + if (!state.compareAndSet(State.INITIAL, State.UNREGISTERED)) + throw new IllegalStateException(String.format("Cannot move to UNREGISTERED state (%s)", state.get())); + } + + public void onRegistration() + { + // This may have been done already if the metadata log replay at start up included our registration + RegistrationStatus.State current = state.get(); + if (current == State.REGISTERED) + return; + + logger.info("This node is registered, moving state to REGISTERED and interrupting any previously established peer connections"); + state.getAndSet(RegistrationStatus.State.REGISTERED); + MessagingService.instance().channelManagers.keySet().forEach(MessagingService.instance()::interruptOutbound); + } +} diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index 2b60c1561d..b184866f57 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -135,7 +135,8 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; public static void initializeAsFirstCMSNode() { InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort(); - ClusterMetadataService.instance().log().bootstrap(addr); + String datacenter = DatabaseDescriptor.getLocator().local().datacenter; + ClusterMetadataService.instance().log().bootstrap(addr, datacenter); ClusterMetadata metadata = ClusterMetadata.current(); assert ClusterMetadataService.state() == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata); diff --git a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java index 8e191307d1..855cce0d36 100644 --- a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java @@ -44,7 +44,6 @@ public class StubClusterMetadataService extends ClusterMetadataService public static StubClusterMetadataService forClientTools() { - DatabaseDescriptor.setLocalDataCenter("DC1"); KeyspaceMetadata ks = DistributedMetadataLogKeyspace.initialMetadata(Collections.singleton("DC1")); DistributedSchema schema = new DistributedSchema(Keyspaces.of(ks)); return new StubClusterMetadataService(new ClusterMetadata(DatabaseDescriptor.getPartitioner(), @@ -54,7 +53,6 @@ public class StubClusterMetadataService extends ClusterMetadataService public static StubClusterMetadataService forClientTools(DistributedSchema initialSchema) { - DatabaseDescriptor.setLocalDataCenter("DC1"); ClusterMetadata metadata = new ClusterMetadata(DatabaseDescriptor.getPartitioner()); metadata = metadata.transformer().with(initialSchema).build().metadata.forceEpoch(Epoch.EMPTY); return new StubClusterMetadataService(metadata); @@ -167,6 +165,7 @@ public class StubClusterMetadataService extends ClusterMetadataService public StubClusterMetadataService build() { if (initial == null) + { initial = new ClusterMetadata(Epoch.EMPTY, partitioner, DistributedSchema.empty(), @@ -176,6 +175,7 @@ public class StubClusterMetadataService extends ClusterMetadataService LockedRanges.EMPTY, InProgressSequences.EMPTY, ImmutableMap.of()); + } return new StubClusterMetadataService(new UniformRangePlacement(), snapshots != null ? snapshots : MetadataSnapshots.NO_OP, LocalLog.logSpec().withInitialState(initial).createLog(), diff --git a/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java b/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java index 4a15925bb9..676915d3f2 100644 --- a/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java +++ b/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java @@ -157,7 +157,7 @@ public class TokenRingUtils public static Collection> getPrimaryRangeForEndpointWithinDC(String keyspace, InetAddressAndPort referenceEndpoint) { ClusterMetadata metadata = ClusterMetadata.current(); - String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(referenceEndpoint); + String localDC = DatabaseDescriptor.getLocator().location(referenceEndpoint).datacenter; Collection localDcNodes = metadata.directory.datacenterEndpoints(localDC); AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy(); diff --git a/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java b/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java index 25aca06f00..f0fced7ae6 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java @@ -34,6 +34,7 @@ import org.apache.cassandra.db.virtual.PeersTable; import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; @@ -50,6 +51,7 @@ import static org.apache.cassandra.tcm.membership.NodeState.BOOTSTRAPPING; import static org.apache.cassandra.tcm.membership.NodeState.BOOT_REPLACING; import static org.apache.cassandra.tcm.membership.NodeState.LEFT; import static org.apache.cassandra.tcm.membership.NodeState.MOVING; +import static org.apache.cassandra.tcm.membership.NodeState.REGISTERED; public class LegacyStateListener implements ChangeListener.Async { @@ -115,8 +117,14 @@ public class LegacyStateListener implements ChangeListener.Async Gossiper.instance.addLocalApplicationState(SCHEMA, StorageService.instance.valueFactory.schema(next.schema.getVersion())); } - - if (next.directory.peerState(change) == LEFT) + if (next.directory.peerState(change) == REGISTERED) + { + // Re-establish any connections made prior to this node registering + InetAddressAndPort endpoint = next.directory.endpoint(change); + logger.info("Peer with address {} has registered, interrupting any previously established connections", endpoint); + MessagingService.instance().interruptOutbound(endpoint); + } + else if (next.directory.peerState(change) == LEFT) { Gossiper.instance.mergeNodeToGossip(change, next, prev.tokenMap.tokens(change)); InetAddressAndPort endpoint = prev.directory.endpoint(change); diff --git a/src/java/org/apache/cassandra/tcm/listeners/SchemaListener.java b/src/java/org/apache/cassandra/tcm/listeners/SchemaListener.java index 2dadbb5007..a506cb1877 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/SchemaListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/SchemaListener.java @@ -26,6 +26,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaDiagnostics; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; public class SchemaListener implements ChangeListener { @@ -44,7 +45,7 @@ public class SchemaListener implements ChangeListener protected void notifyInternal(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot, boolean loadSSTables) { - if (!fromSnapshot && next.schema.lastModified().equals(prev.schema.lastModified())) + if (next.epoch.isAfter(Epoch.FIRST) && !fromSnapshot && next.schema.lastModified().equals(prev.schema.lastModified())) return; next.schema.initializeKeyspaceInstances(prev.schema, loadSSTables); } diff --git a/src/java/org/apache/cassandra/tcm/log/LocalLog.java b/src/java/org/apache/cassandra/tcm/log/LocalLog.java index 36b6e41011..6e54e4657f 100644 --- a/src/java/org/apache/cassandra/tcm/log/LocalLog.java +++ b/src/java/org/apache/cassandra/tcm/log/LocalLog.java @@ -279,14 +279,24 @@ public abstract class LocalLog implements Closeable entryFilters = Lists.newCopyOnWriteArrayList(); } - public void bootstrap(InetAddressAndPort addr) + @VisibleForTesting + public void unsafeBootstrapForTesting(InetAddressAndPort addr) + { + bootstrap(addr, ""); + } + + /** + * + * @param addr + * @param datacenter + */ + public void bootstrap(InetAddressAndPort addr, String datacenter) { ClusterMetadata metadata = metadata(); assert metadata.epoch.isBefore(FIRST) : String.format("Metadata epoch %s should be before first", metadata.epoch); - Transformation transform = PreInitialize.withFirstCMS(addr); + Transformation transform = PreInitialize.withFirstCMS(addr, datacenter); append(new Entry(Entry.Id.NONE, FIRST, transform)); - waitForHighestConsecutive(); - metadata = metadata(); + metadata = waitForHighestConsecutive(); assert metadata.epoch.is(Epoch.FIRST) : String.format("Epoch: %s. CMS: %s", metadata.epoch, metadata.fullCMSMembers()); } diff --git a/src/java/org/apache/cassandra/tcm/membership/Directory.java b/src/java/org/apache/cassandra/tcm/membership/Directory.java index 2b3dc967e4..87a6bde053 100644 --- a/src/java/org/apache/cassandra/tcm/membership/Directory.java +++ b/src/java/org/apache/cassandra/tcm/membership/Directory.java @@ -38,6 +38,7 @@ import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; + import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataValue; import org.apache.cassandra.tcm.serialization.MetadataSerializer; diff --git a/src/java/org/apache/cassandra/tcm/membership/Location.java b/src/java/org/apache/cassandra/tcm/membership/Location.java index 7e38db79cb..faf8230d94 100644 --- a/src/java/org/apache/cassandra/tcm/membership/Location.java +++ b/src/java/org/apache/cassandra/tcm/membership/Location.java @@ -30,6 +30,7 @@ import org.apache.cassandra.tcm.serialization.Version; public class Location { public static final Serializer serializer = new Serializer(); + public static final Location UNKNOWN = new Location("UNKNOWN_DC", "UNKNOWN_RACK"); public final String datacenter; public final String rack; @@ -40,6 +41,11 @@ public class Location this.rack = rack; } + public boolean sameDatacenter(Location other) + { + return datacenter.equals(other.datacenter); + } + @Override public boolean equals(Object o) { diff --git a/src/java/org/apache/cassandra/tcm/membership/NodeAddresses.java b/src/java/org/apache/cassandra/tcm/membership/NodeAddresses.java index 53f2d1acba..dcbb90eb68 100644 --- a/src/java/org/apache/cassandra/tcm/membership/NodeAddresses.java +++ b/src/java/org/apache/cassandra/tcm/membership/NodeAddresses.java @@ -84,7 +84,6 @@ public class NodeAddresses public boolean conflictsWith(NodeAddresses other) { return broadcastAddress.equals(other.broadcastAddress) || - localAddress.equals(other.localAddress) || nativeAddress.equals(other.nativeAddress); } diff --git a/src/java/org/apache/cassandra/tcm/sequences/Move.java b/src/java/org/apache/cassandra/tcm/sequences/Move.java index 09811fba80..99ada06f93 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/Move.java +++ b/src/java/org/apache/cassandra/tcm/sequences/Move.java @@ -367,8 +367,9 @@ public class Move extends MultiStepOperation // if we are not running with strict consistency, try to find other sources for streaming if (needsRelaxedSources.get()) { - for (Replica source : DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), - oldOwners.forRange(destination.range()).get())) + for (Replica source : DatabaseDescriptor.getNodeProximity() + .sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), + oldOwners.forRange(destination.range()).get())) { if (fd.isAlive(source.endpoint()) && !source.endpoint().equals(destination.endpoint())) { diff --git a/src/java/org/apache/cassandra/tcm/serialization/Version.java b/src/java/org/apache/cassandra/tcm/serialization/Version.java index ed601009f6..55bbbfb04e 100644 --- a/src/java/org/apache/cassandra/tcm/serialization/Version.java +++ b/src/java/org/apache/cassandra/tcm/serialization/Version.java @@ -48,6 +48,7 @@ public enum Version V4(4), /** * - AlterSchema includes execution timestamp + * - PreInitialize includes datacenter (affects local serialization on first CMS node only) */ V5(5), diff --git a/src/java/org/apache/cassandra/tcm/transformations/Register.java b/src/java/org/apache/cassandra/tcm/transformations/Register.java index f53eb9a1f6..389f62c698 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/Register.java +++ b/src/java/org/apache/cassandra/tcm/transformations/Register.java @@ -31,7 +31,6 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Transformation; @@ -96,30 +95,37 @@ public class Register implements Transformation return register(nodeAddresses, NodeVersion.CURRENT); } + @VisibleForTesting + public static NodeId register(NodeAddresses nodeAddresses, Location location) + { + return register(nodeAddresses, location, NodeVersion.CURRENT); + } + @VisibleForTesting public static NodeId register(NodeAddresses nodeAddresses, NodeVersion nodeVersion) { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - Location location = new Location(snitch.getLocalDatacenter(), snitch.getLocalRack()); + return register(nodeAddresses, DatabaseDescriptor.getLocator().local(), nodeVersion); + } + private static NodeId register(NodeAddresses nodeAddresses, Location location, NodeVersion nodeVersion) + { ClusterMetadata metadata = ClusterMetadata.current(); + DatabaseDescriptor.getInitialLocationProvider().validate(metadata); NodeId nodeId = metadata.directory.peerId(nodeAddresses.broadcastAddress); if (nodeId == null || metadata.directory.peerState(nodeId) == NodeState.LEFT) { if (nodeId != null) - ClusterMetadataService.instance() - .commit(new Unregister(nodeId, EnumSet.of(NodeState.LEFT))); - nodeId = ClusterMetadataService.instance() - .commit(new Register(nodeAddresses, location, nodeVersion)) - .directory - .peerId(nodeAddresses.broadcastAddress); + ClusterMetadataService.instance().commit(new Unregister(nodeId, EnumSet.of(NodeState.LEFT))); + + Register registration = new Register(nodeAddresses, location, nodeVersion); + nodeId = ClusterMetadataService.instance().commit(registration).directory.peerId(nodeAddresses.broadcastAddress); } else { throw new IllegalStateException(String.format("A node with address %s already exists, cancelling join. Use cassandra.replace_address if you want to replace this node.", nodeAddresses.broadcastAddress)); } - logger.info("Registering with endpoint {}", nodeAddresses.broadcastAddress); + logger.info("Registered with endpoint {}, node id: {}", nodeAddresses.broadcastAddress, nodeId); return nodeId; } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java index 50aa3aef42..ed25d33cdf 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java @@ -26,6 +26,9 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; +import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; @@ -41,25 +44,27 @@ public class PreInitialize implements Transformation public static Serializer serializer = new Serializer(); public final InetAddressAndPort addr; + public final String datacenter; - private PreInitialize(InetAddressAndPort addr) + private PreInitialize(InetAddressAndPort addr, String datacenter) { this.addr = addr; + this.datacenter = datacenter; } public static PreInitialize forTesting() { - return new PreInitialize(null); + return new PreInitialize(null, null); } public static PreInitialize blank() { - return new PreInitialize(null); + return new PreInitialize(null, null); } - public static PreInitialize withFirstCMS(InetAddressAndPort addr) + public static PreInitialize withFirstCMS(InetAddressAndPort addr, String datacenter) { - return new PreInitialize(addr); + return new PreInitialize(addr, datacenter); } public Kind kind() @@ -72,8 +77,24 @@ public class PreInitialize implements Transformation assert metadata.epoch.isBefore(Epoch.FIRST); ClusterMetadata.Transformer transformer = metadata.transformer(); + if (addr != null) { + // If addr != null, then this is being executed on the peer which is actually initializing the log + // for the very first time. + + // addr and datacenter are only used to bootstrap the replication of the distributed metatada + // keyspace on the first CMS node. They are never serialized into the distributed metadata log or + // passed to any other peer. + // + // PRE_INITIALIZE_CMS @ Epoch.FIRST, must be followed in the log by INITIALIZE_CMS @ (Epoch.FIRST + 1). + // The serialization of INITIALIZE_CMS includes the full ClusterMetadata at that point, which is + // obviously minimal, but will necessarily include the distributed metadata keyspace definition with + // the replication settings bootstrapped by PRE_INITIALIZE. This full ClusterMetadata becomes the + // starting point upon which further log entries are applied. So this means that once INITIALIZE_CMS + // has been committed to the log, the actual content of PRE_INITIALIZE_CMS is irrelevant, even on + // the first CMS node if it happens to replay it from its local storage after a restart. + DataPlacement.Builder dataPlacementBuilder = DataPlacement.builder(); Replica replica = new Replica(addr, MetaStrategy.partitioner.getMinimumToken(), @@ -81,10 +102,18 @@ public class PreInitialize implements Transformation true); dataPlacementBuilder.reads.withReplica(Epoch.FIRST, replica); dataPlacementBuilder.writes.withReplica(Epoch.FIRST, replica); - DataPlacements initialPlacement = metadata.placements.unbuild().with(ReplicationParams.meta(metadata), dataPlacementBuilder.build()).build(); + DataPlacements initialPlacement = metadata.placements.unbuild() + .with(ReplicationParams.simpleMeta(1, datacenter), + dataPlacementBuilder.build()).build(); transformer.with(initialPlacement); + // re-initialise the schema distributed metadata keyspace so it gets the + // correct replication settings based on the DC of the initial CMS node + Keyspaces updated = metadata.schema.getKeyspaces() + .withAddedOrReplaced(DistributedMetadataLogKeyspace.initialMetadata(datacenter)); + transformer.with(new DistributedSchema(updated)); } + ClusterMetadata.Transformer.Transformed transformed = transformer.build(); metadata = transformed.metadata.forceEpoch(Epoch.FIRST); assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch; @@ -94,12 +123,10 @@ public class PreInitialize implements Transformation public String toString() { - return "PreInitialize{" + - "addr=" + addr + - '}'; + return "PreInitialize"; } - static class Serializer implements AsymmetricMetadataSerializer + public static class Serializer implements AsymmetricMetadataSerializer { public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException @@ -108,7 +135,9 @@ public class PreInitialize implements Transformation PreInitialize bcms = (PreInitialize)t; out.writeBoolean(bcms.addr != null); if (bcms.addr != null) - InetAddressAndPort.MetadataSerializer.serializer.serialize(((PreInitialize)t).addr, out, version); + InetAddressAndPort.MetadataSerializer.serializer.serialize(bcms.addr, out, version); + if (bcms.datacenter != null && version.isAtLeast(Version.V5)) + out.writeUTF(bcms.datacenter); } public PreInitialize deserialize(DataInputPlus in, Version version) throws IOException @@ -118,7 +147,8 @@ public class PreInitialize implements Transformation return PreInitialize.blank(); InetAddressAndPort addr = InetAddressAndPort.MetadataSerializer.serializer.deserialize(in, version); - return new PreInitialize(addr); + String datacenter = version.isAtLeast(Version.V5) ? in.readUTF() : ""; + return new PreInitialize(addr, datacenter); } public long serializedSize(Transformation t, Version version) @@ -126,7 +156,11 @@ public class PreInitialize implements Transformation PreInitialize bcms = (PreInitialize)t; long size = TypeSizes.sizeof(bcms.addr != null); - return size + (bcms.addr != null ? InetAddressAndPort.MetadataSerializer.serializer.serializedSize(((PreInitialize)t).addr, version) : 0); + if (bcms.addr != null) + size += InetAddressAndPort.MetadataSerializer.serializer.serializedSize(bcms.addr, version); + if (bcms.datacenter != null && version.isAtLeast(Version.V5)) + size += TypeSizes.sizeof(bcms.datacenter); + return size; } } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 8191bb73ec..d0ba27ad51 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -92,6 +92,7 @@ import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.hints.HintsServiceMBean; import org.apache.cassandra.locator.DynamicEndpointSnitchMBean; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; +import org.apache.cassandra.locator.LocationInfoMBean; import org.apache.cassandra.metrics.CIDRAuthorizerMetrics; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.StorageMetrics; @@ -1219,6 +1220,18 @@ public class NodeProbe implements AutoCloseable } } + public LocationInfoMBean getLocationInfoProxy() + { + try + { + return JMX.newMBeanProxy(mbeanServerConn, new ObjectName("org.apache.cassandra.db:type=LocationInfo"), LocationInfoMBean.class); + } + catch (MalformedObjectNameException e) + { + throw new RuntimeException(e); + } + } + public ColumnFamilyStoreMBean getCfsProxy(String ks, String cf) { ColumnFamilyStoreMBean cfsProxy = null; diff --git a/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java b/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java index 23100ea74e..1904eeb50e 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java +++ b/src/java/org/apache/cassandra/tools/nodetool/DescribeCluster.java @@ -28,6 +28,7 @@ import com.google.common.collect.ArrayListMultimap; import io.airlift.airline.Command; import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.LocationInfoMBean; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; @@ -46,15 +47,19 @@ public class DescribeCluster extends NodeToolCmd // display cluster name, snitch and partitioner out.println("Cluster Information:"); out.println("\tName: " + probe.getClusterName()); - String snitch = probe.getEndpointSnitchInfoProxy().getSnitchName(); + LocationInfoMBean locationInfoProxy = probe.getLocationInfoProxy(); + String nodeProximityName = locationInfoProxy.getNodeProximityName(); boolean dynamicSnitchEnabled = false; - if (snitch.equals(DynamicEndpointSnitch.class.getName())) + boolean legacySnitchAdapter = locationInfoProxy.hasLegacySnitchAdapter(); + if (nodeProximityName.equals(DynamicEndpointSnitch.class.getName())) { - snitch = probe.getDynamicEndpointSnitchInfoProxy().getSubsnitchClassName(); + nodeProximityName = probe.getDynamicEndpointSnitchInfoProxy().getSubsnitchClassName(); dynamicSnitchEnabled = true; } - out.println("\tSnitch: " + snitch); + out.println("\tSnitch: " + nodeProximityName); out.println("\tDynamicEndPointSnitch: " + (dynamicSnitchEnabled ? "enabled" : "disabled")); + out.println("\tNodeProximity: " + nodeProximityName); + out.println("\tLegacySnitchAdapter: " + (legacySnitchAdapter ? "enabled" : "disabled")); out.println("\tPartitioner: " + probe.getPartitioner()); // display schema version for each node diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index d52cfeb30b..d1eb0798d8 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -224,6 +224,13 @@ public class FBUtilities return localInetAddressAndPort; } + public static void setLocalAddress(InetAddress localAddress) + { + localInetAddress = localAddress; + // null out localInetAddressAndPort, it will be re-initalized next time it's accessed + localInetAddressAndPort = null; + } + /** * Retrieve just the broadcast address but not the port. This is almost always the wrong thing to be using because * it's ambiguous since you need the address and port to identify a node. You want getBroadcastAddressAndPort diff --git a/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java b/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java index e41b3eca52..9e5e51fa43 100644 --- a/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java @@ -43,7 +43,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; -import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort; +import static org.apache.cassandra.distributed.impl.TestEndpointCache.toCassandraInetAddressAndPort; public class GossipHelper { diff --git a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestInitialLocationProvider.java b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestInitialLocationProvider.java new file mode 100644 index 0000000000..af413797ac --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestInitialLocationProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.impl; + + +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.locator.InitialLocationProvider; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.utils.FBUtilities; + +public class DistributedTestInitialLocationProvider implements InitialLocationProvider +{ + private static NetworkTopology mapping = null; + public static void assign(NetworkTopology topology) + { + mapping = topology; + } + + @Override + public Location initialLocation() + { + String dc = mapping.localDC(FBUtilities.getBroadcastAddressAndPort()); + String rack = mapping.localRack(FBUtilities.getBroadcastAddressAndPort()); + return new Location(dc, rack); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java b/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java deleted file mode 100644 index 678af72b7f..0000000000 --- a/test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.distributed.impl; - -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.cassandra.config.Config; -import org.apache.cassandra.db.SystemKeyspace; -import org.apache.cassandra.distributed.shared.NetworkTopology; -import org.apache.cassandra.gms.ApplicationState; -import org.apache.cassandra.gms.EndpointState; -import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.FBUtilities; - -public class DistributedTestSnitch extends AbstractNetworkTopologySnitch -{ - private static NetworkTopology mapping = null; - private static final Map cache = new ConcurrentHashMap<>(); - private static final Map cacheInverse = new ConcurrentHashMap<>(); - - public static InetAddressAndPort toCassandraInetAddressAndPort(InetSocketAddress addressAndPort) - { - InetAddressAndPort m = cacheInverse.get(addressAndPort); - if (m == null) - { - m = InetAddressAndPort.getByAddressOverrideDefaults(addressAndPort.getAddress(), addressAndPort.getPort()); - cache.put(m, addressAndPort); - } - return m; - } - - public static InetSocketAddress fromCassandraInetAddressAndPort(InetAddressAndPort addressAndPort) - { - InetSocketAddress m = cache.get(addressAndPort); - if (m == null) - { - m = NetworkTopology.addressAndPort(addressAndPort.getAddress(), addressAndPort.getPort()); - cache.put(addressAndPort, m); - } - return m; - } - - private Map> savedEndpoints; - private static final String DEFAULT_DC = "UNKNOWN_DC"; - private static final String DEFAULT_RACK = "UNKNOWN_RCK"; // TODO must be =< 12 chars to preserve nodetool output required by tests - - public String getRack(InetAddress endpoint) - { - int storage_port = Config.getOverrideLoadConfig().get().storage_port; - return getRack(InetAddressAndPort.getByAddressOverrideDefaults(endpoint, storage_port)); - } - - public String getRack(InetAddressAndPort endpoint) - { - assert mapping != null : "network topology must be assigned before using snitch"; - return maybeGetFromEndpointState(mapping.localRack(fromCassandraInetAddressAndPort(endpoint)), endpoint, ApplicationState.RACK, DEFAULT_RACK); - } - - public String getDatacenter(InetAddress endpoint) - { - int storage_port = Config.getOverrideLoadConfig().get().storage_port; - return getDatacenter(InetAddressAndPort.getByAddressOverrideDefaults(endpoint, storage_port)); - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - assert mapping != null : "network topology must be assigned before using snitch"; - return maybeGetFromEndpointState(mapping.localDC(fromCassandraInetAddressAndPort(endpoint)), endpoint, ApplicationState.DC, DEFAULT_DC); - } - - // Here, the logic is slightly different from what we have in GossipingPropertyFileSnitch since we have a different - // goal. Passed argument (topology that was set on the node) overrides anything that is passed elsewhere. - private String maybeGetFromEndpointState(String current, InetAddressAndPort endpoint, ApplicationState state, String defaultValue) - { - if (current != null) - return current; - - EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint); - if (epState == null || epState.getApplicationState(state) == null) - { - if (savedEndpoints == null) - savedEndpoints = SystemKeyspace.loadDcRackInfo(); - if (savedEndpoints.containsKey(endpoint)) - return savedEndpoints.get(endpoint).get("data_center"); - - return defaultValue; - } - - return epState.getApplicationState(state).value; - } - - static void assign(NetworkTopology newMapping) - { - mapping = new NetworkTopology(newMapping); - } - - public void gossiperStarting() - { - super.gossiperStarting(); - - Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT, - StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort())); - Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, - StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress())); - } -} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index 1d2272ad4d..df8acbfbd7 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -140,6 +140,7 @@ import org.apache.cassandra.streaming.async.NettyStreamingChannel; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.EpochAwareDebounce; +import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.tcm.Startup; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; @@ -178,8 +179,8 @@ import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.JMX; import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL; import static org.apache.cassandra.distributed.api.Feature.NETWORK; -import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.fromCassandraInetAddressAndPort; -import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort; +import static org.apache.cassandra.distributed.impl.TestEndpointCache.fromCassandraInetAddressAndPort; +import static org.apache.cassandra.distributed.impl.TestEndpointCache.toCassandraInetAddressAndPort; import static org.apache.cassandra.net.Verb.BATCH_STORE_REQ; import static org.apache.cassandra.service.CassandraDaemon.logSystemInfo; @@ -622,7 +623,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { assert config.networkTopology().contains(config.broadcastAddress()) : String.format("Network topology %s doesn't contain the address %s", config.networkTopology(), config.broadcastAddress()); - DistributedTestSnitch.assign(config.networkTopology()); + DistributedTestInitialLocationProvider.assign(config.networkTopology()); CassandraDaemon.getInstanceForTesting().activate(false); // TODO: filters won't work for the messages dispatched during startup registerInboundFilter(cluster); @@ -706,7 +707,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance assert config.networkTopology().contains(config.broadcastAddress()) : String.format("Network topology %s doesn't contain the address %s", config.networkTopology(), config.broadcastAddress()); - DistributedTestSnitch.assign(config.networkTopology()); + DistributedTestInitialLocationProvider.assign(config.networkTopology()); DatabaseDescriptor.daemonInitialization(); if (config.has(JMX)) startJmx(); @@ -811,6 +812,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance Schema.instance.saveSystemKeyspace(); ClusterMetadataService.instance().processor().fetchLogAndWait(); NodeId self = Register.maybeRegister(); + RegistrationStatus.instance.onRegistration(); boolean joinRing = config.get(Constants.KEY_DTEST_JOIN_RING) == null || (boolean) config.get(Constants.KEY_DTEST_JOIN_RING); if (ClusterMetadata.current().directory.peerState(self) != NodeState.JOINED && joinRing) { diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java index 7f16d05680..8afd90faf7 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java @@ -37,6 +37,7 @@ import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.upgrade.UpgradeTestBase; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.NetworkTopologyProximity; import org.apache.cassandra.locator.SimpleSeedProvider; public class InstanceConfig implements IInstanceConfig @@ -105,7 +106,8 @@ public class InstanceConfig implements IInstanceConfig .set("commitlog_sync_period_in_ms", 10000) .set("storage_port", storage_port) .set("native_transport_port", native_transport_port) - .set("endpoint_snitch", DistributedTestSnitch.class.getName()) + .set("initial_location_provider", DistributedTestInitialLocationProvider.class.getName()) + .set("node_proximity", NetworkTopologyProximity.class.getName()) .set("seed_provider", new ParameterizedClass(SimpleSeedProvider.class.getName(), Collections.singletonMap("seeds", seedIp + ':' + seedPort))) .set("discovery_timeout", "3s") @@ -184,7 +186,7 @@ public class InstanceConfig implements IInstanceConfig @Override public InetSocketAddress broadcastAddress() { - return DistributedTestSnitch.fromCassandraInetAddressAndPort(getBroadcastAddressAndPort()); + return TestEndpointCache.fromCassandraInetAddressAndPort(getBroadcastAddressAndPort()); } public void unsetBroadcastAddressAndPort() @@ -345,12 +347,22 @@ public class InstanceConfig implements IInstanceConfig public InstanceConfig forVersion(Semver version) { // Versions before 4.0 need to set 'seed_provider' without specifying the port - if (UpgradeTestBase.v40.compareTo(version) < 0) + // Versions before 5.0 need to set 'endpoint_snitch', not initial_location_provider + node_proximity + if (version.compareTo(UpgradeTestBase.v51) >= 0) return this; - else - return new InstanceConfig(this) - .set("seed_provider", new ParameterizedClass(SimpleSeedProvider.class.getName(), - Collections.singletonMap("seeds", "127.0.0.1"))); + + InstanceConfig config = new InstanceConfig(this); + config.remove("initial_location_provider"); + config.remove("node_proximity"); + config.set("endpoint_snitch", "org.apache.cassandra.distributed.impl.DistributedTestSnitch"); + + // 4.0+ has seed_provider without port + if (version.compareTo(UpgradeTestBase.v40) >= 0) + return config; + + config.set("seed_provider", new ParameterizedClass(SimpleSeedProvider.class.getName(), + Collections.singletonMap("seeds", "127.0.0.1"))); + return config; } public String toString() diff --git a/test/distributed/org/apache/cassandra/distributed/impl/TestEndpointCache.java b/test/distributed/org/apache/cassandra/distributed/impl/TestEndpointCache.java new file mode 100644 index 0000000000..a7d6b06be1 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/impl/TestEndpointCache.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.impl; + +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.locator.InetAddressAndPort; + +public class TestEndpointCache +{ + private static final Map cache = new ConcurrentHashMap<>(); + private static final Map cacheInverse = new ConcurrentHashMap<>(); + + public static InetAddressAndPort toCassandraInetAddressAndPort(InetSocketAddress addressAndPort) + { + InetAddressAndPort m = cacheInverse.get(addressAndPort); + if (m == null) + { + m = InetAddressAndPort.getByAddressOverrideDefaults(addressAndPort.getAddress(), addressAndPort.getPort()); + cacheInverse.put(addressAndPort, m); + } + return m; + } + + public static InetSocketAddress fromCassandraInetAddressAndPort(InetAddressAndPort addressAndPort) + { + InetSocketAddress m = cache.get(addressAndPort); + if (m == null) + { + m = NetworkTopology.addressAndPort(addressAndPort.getAddress(), addressAndPort.getPort()); + cache.put(addressAndPort, m); + } + return m; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java b/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java index c111946bed..377ac19880 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java @@ -42,7 +42,7 @@ import org.apache.cassandra.utils.FBUtilities; import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Collections.singleton; -import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort; +import static org.apache.cassandra.distributed.impl.TestEndpointCache.toCassandraInetAddressAndPort; import static org.apache.cassandra.locator.InetAddressAndPort.getByAddress; public class UnsafeGossipHelper diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java index 2f9d628d54..bf6a0879ce 100644 --- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java +++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java @@ -33,9 +33,11 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.FailureDetectorMBean; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.hints.HintsService; +import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.DynamicEndpointSnitchMBean; import org.apache.cassandra.locator.EndpointSnitchInfo; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; +import org.apache.cassandra.locator.SnitchAdapter; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.ActiveRepairService; @@ -104,7 +106,8 @@ public class InternalNodeProbe extends NodeProbe @Override public DynamicEndpointSnitchMBean getDynamicEndpointSnitchInfoProxy() { - return (DynamicEndpointSnitchMBean) DatabaseDescriptor.createEndpointSnitch(true, DatabaseDescriptor.getRawConfig().endpoint_snitch); + // TODO At some point we should change this to use modern config e.g. Locator and InitialLocationProvider + return new DynamicEndpointSnitch(new SnitchAdapter(DatabaseDescriptor.createEndpointSnitch(DatabaseDescriptor.getRawConfig().endpoint_snitch))); } public CacheServiceMBean getCacheServiceMBean() diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index 4aeb630b66..313dde0c54 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -77,7 +77,6 @@ import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Commit; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; -import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.ReplicaGroups; import org.apache.cassandra.utils.Isolated; @@ -89,7 +88,7 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_ import static org.apache.cassandra.config.CassandraRelevantProperties.BROADCAST_INTERVAL_MS; import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS_FIRST_BOOT; import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY; -import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort; +import static org.apache.cassandra.distributed.impl.TestEndpointCache.toCassandraInetAddressAndPort; import static org.assertj.core.api.Assertions.assertThat; /** diff --git a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java index e66a85a3f2..b7f7c5ffb5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/GossipTest.java @@ -56,7 +56,7 @@ import static org.apache.cassandra.distributed.action.GossipHelper.withProperty; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; import static org.apache.cassandra.distributed.api.TokenSupplier.evenlyDistributedTokens; -import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort; +import static org.apache.cassandra.distributed.impl.TestEndpointCache.toCassandraInetAddressAndPort; import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCommit; import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart; import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked; diff --git a/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java b/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java index d72180f902..9f242b0212 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/IPMembershipTest.java @@ -34,6 +34,8 @@ import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.impl.InstanceConfig; import org.apache.cassandra.distributed.shared.ClusterUtils; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.locator.NoOpProximity; +import org.apache.cassandra.locator.SimpleLocationProvider; import org.apache.cassandra.tools.ToolRunner; import org.assertj.core.api.Assertions; @@ -83,8 +85,9 @@ public class IPMembershipTest extends TestBaseImpl { try (Cluster cluster = Cluster.build(3) .withConfig(c -> c.with(Feature.GOSSIP, Feature.NATIVE_PROTOCOL) - // disable DistributedTestSnitch as it tries to query before we setup - .set("endpoint_snitch", "org.apache.cassandra.locator.SimpleSnitch")) + // disable DistributedTestInitialLocationProvider as it tries to query before we setup + .set("node_proximity", NoOpProximity.class.getName()) + .set("initial_location_provider", SimpleLocationProvider.class.getName())) .start()) { IInvokableInstance nodeToReplace = cluster.get(3); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReconnectToInternalIPTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReconnectToInternalIPTest.java new file mode 100644 index 0000000000..6d798d5aa9 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/ReconnectToInternalIPTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector; +import org.apache.cassandra.locator.Ec2MultiRegionAddressConfig; +import org.apache.cassandra.locator.Ec2MultiRegionSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.InitialLocationProvider; +import org.apache.cassandra.locator.NetworkTopologyProximity; +import org.apache.cassandra.locator.SnitchProperties; +import org.apache.cassandra.tcm.membership.Location; +import org.awaitility.Awaitility; + +import static org.apache.cassandra.locator.Ec2LocationProvider.ZONE_NAME_QUERY; +import static org.apache.cassandra.locator.Ec2MultiRegionAddressConfig.PRIVATE_IP_QUERY; +import static org.apache.cassandra.locator.Ec2MultiRegionAddressConfig.PUBLIC_IP_QUERY; +import static org.apache.cassandra.utils.Pair.create; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ReconnectToInternalIPTest extends TestBaseImpl +{ + @Test + public void testWithSnitchConfig() throws Exception + { + try (Cluster cluster = init(builder().withNodes(4) + .withConfig(c -> c.set("endpoint_snitch", TestMultiRegionSnitch.class.getName()) + .set("node_proximity", null) + .set("initial_location_provider", null) + .set("listen_on_broadcast_address", true) + .with(Feature.NETWORK, Feature.GOSSIP)) + .start())) + { + doTest(cluster); + } + } + + @Test + public void testWithModernConfig() throws Exception + { + try (Cluster cluster = init(builder().withNodes(4) + .withConfig(c -> c.set("endpoint_snitch", null) + .set("node_proximity", NetworkTopologyProximity.class.getName()) + .set("initial_location_provider", TestMultiRegionLocationProvider.class.getName()) + .set("listen_on_broadcast_address", true) + .set("addresses_config", TestMultiRegionAddressConfig.class.getName()) + .set("prefer_local_connections", true) + .with(Feature.NETWORK, Feature.GOSSIP)) + .start())) + { + doTest(cluster); + } + } + + private static void doTest(Cluster cluster) throws TimeoutException + { + cluster.schemaChange(withKeyspace("create table %s.tbl (id int primary key)")); + cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (id) values (1)"), ConsistencyLevel.ALL); + // node1 should only reconnect to node2, but currently this is still driven by gossip so may not happen immediately + InetSocketAddress ep = cluster.get(2).config().broadcastAddress(); + String pattern = "Initiated reconnect to an Internal IP "+toInternalIp(ep)+" for the " + ep; + cluster.get(1).logs().watchFor(Duration.ofSeconds(10), pattern); + + // node1 should not reconnect to node3 or node4 + for (int i = 3; i <= 4; i++) + { + ep = cluster.get(i).config().broadcastAddress(); + pattern = "Initiated reconnect to an Internal IP "+toInternalIp(ep)+" for the " + ep; + assertTrue(cluster.get(1).logs().grep(pattern).getResult().isEmpty()); + } + + cluster.forEach(inst -> inst.runOnInstance(() -> { + for (InetAddressAndPort endpoint : Gossiper.instance.endpointStateMap.keySet()) + { + InetAddressAndPort internal = toInternalIp(endpoint); + Awaitility.await() + .atMost(30L, TimeUnit.SECONDS) + .until(() -> Gossiper.instance.getApplicationState(endpoint, ApplicationState.INTERNAL_ADDRESS_AND_PORT) != null); + InetAddressAndPort fromGossip = InetAddressAndPort.getByNameUnchecked(Gossiper.instance.getApplicationState(endpoint, ApplicationState.INTERNAL_ADDRESS_AND_PORT)); + assertEquals(internal, fromGossip); + } + })); + } + + public static class TestMultiRegionLocationProvider implements InitialLocationProvider + { + @Override + public Location initialLocation() + { + InetAddressAndPort configuredBA = InetAddressAndPort.getByNameUnchecked(DatabaseDescriptor.getRawConfig().broadcast_address); + byte lastByte = configuredBA.addressBytes[configuredBA.addressBytes.length - 1]; + String zone = (lastByte == 1 || lastByte == 2) ? "us-east-1" : "us-west-1"; + return new Location(zone, zone + 'a'); + } + } + + public static class TestMultiRegionAddressConfig extends Ec2MultiRegionAddressConfig + { + public TestMultiRegionAddressConfig() throws IOException + { + super(new TestCloudMetadataConnector()); + } + } + + public static class TestMultiRegionSnitch extends Ec2MultiRegionSnitch + { + public TestMultiRegionSnitch() throws IOException, ConfigurationException + { + super(new TestCloudMetadataConnector()); + } + } + + public static class TestCloudMetadataConnector extends AbstractCloudMetadataServiceConnector + { + public TestCloudMetadataConnector() + { + super(new SnitchProperties(create(METADATA_URL_PROPERTY, "http://apache.org"))); + } + + @Override + public String apiCall(String url, + String query, + String method, + Map extraHeaders, + int expectedResponseCode) + { + InetAddressAndPort configuredBA = InetAddressAndPort.getByNameUnchecked(DatabaseDescriptor.getRawConfig().broadcast_address); + switch (query) + { + case ZONE_NAME_QUERY: + // "us-east-1a" + byte lastByte = configuredBA.addressBytes[configuredBA.addressBytes.length - 1]; + if (lastByte == 1 || lastByte == 2) + return "us-east-1a"; + else + return "us-west-1a"; + case PUBLIC_IP_QUERY: + // public ip is configured ip "+ 4" + return toPublicIp(configuredBA).getHostAddress(false); + case PRIVATE_IP_QUERY: + // private ip is configured ip "+ 8" + return toInternalIp(configuredBA).getHostAddress(false); + default: + throw new AssertionError("Bad query: " + query); + } + } + } + + private static InetAddressAndPort toInternalIp(InetSocketAddress ep) + { + return convertIp(ep, 8); + } + + private static InetAddressAndPort toPublicIp(InetSocketAddress ep) + { + return convertIp(ep, 4); + } + + private static InetAddressAndPort convertIp(InetSocketAddress ep, int offset) + { + byte lastByte = ep.getAddress().getAddress()[ep.getAddress().getAddress().length - 1]; + return InetAddressAndPort.getByNameUnchecked("127.0.0." + (lastByte + offset)+":"+ep.getPort()); + } + +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/BounceResetHostIdTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/BounceResetHostIdTest.java index e593a71b68..0a8a31e3a2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/BounceResetHostIdTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/BounceResetHostIdTest.java @@ -32,6 +32,8 @@ import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.shared.AssertUtils; import org.apache.cassandra.distributed.shared.ClusterUtils; import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.locator.NoOpProximity; +import org.apache.cassandra.locator.SimpleLocationProvider; import org.apache.cassandra.tcm.membership.NodeId; import static org.apache.cassandra.distributed.shared.AssertUtils.row; @@ -44,8 +46,9 @@ public class BounceResetHostIdTest extends TestBaseImpl { try (Cluster cluster = builder().withNodes(3) .withConfig(c -> c.with(Feature.GOSSIP, Feature.NATIVE_PROTOCOL) - // disable DistributedTestSnitch as it tries to query before we setup - .set("endpoint_snitch", "org.apache.cassandra.locator.SimpleSnitch")) + // disable DistributedTestInitialLocationProvider as it tries to query before we setup + .set("node_proximity", NoOpProximity.class.getName()) + .set("initial_location_provider", SimpleLocationProvider.class.getName())) .createWithoutStarting()) { // This test relies on node IDs being in the same order as IP addresses @@ -97,8 +100,9 @@ public class BounceResetHostIdTest extends TestBaseImpl { try (Cluster cluster = builder().withNodes(3) .withConfig(c -> c.with(Feature.GOSSIP, Feature.NATIVE_PROTOCOL) - // disable DistributedTestSnitch as it tries to query before we setup - .set("endpoint_snitch", "org.apache.cassandra.locator.SimpleSnitch")) + // disable DistributedTestInitialLocationProvider as it tries to query before we setup + .set("node_proximity", NoOpProximity.class.getName()) + .set("initial_location_provider", SimpleLocationProvider.class.getName())) .createWithoutStarting()) { // This test relies on node IDs being in the same order as IP addresses diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CMSMembershipMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/CMSMembershipMetricsTest.java index a165f39848..58a33400b7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CMSMembershipMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CMSMembershipMetricsTest.java @@ -26,7 +26,7 @@ import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.IInvokableInstance; -import org.apache.cassandra.distributed.impl.DistributedTestSnitch; +import org.apache.cassandra.distributed.impl.TestEndpointCache; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.metrics.TCMMetrics; @@ -106,14 +106,14 @@ public class CMSMembershipMetricsTest extends TestBaseImpl private void markDown(IInvokableInstance down, IInvokableInstance inst) { InetSocketAddress node3Address = down.config().broadcastAddress(); - inst.runOnInstance(() -> FailureDetector.instance.forceConviction(DistributedTestSnitch.toCassandraInetAddressAndPort(node3Address))); + inst.runOnInstance(() -> FailureDetector.instance.forceConviction(TestEndpointCache.toCassandraInetAddressAndPort(node3Address))); } private void markUp(IInvokableInstance down, IInvokableInstance inst) { InetSocketAddress downAddress = down.config().broadcastAddress(); - inst.runOnInstance(() -> FailureDetector.instance.report(DistributedTestSnitch.toCassandraInetAddressAndPort(downAddress))); + inst.runOnInstance(() -> FailureDetector.instance.report(TestEndpointCache.toCassandraInetAddressAndPort(downAddress))); Awaitility.waitAtMost(10, TimeUnit.SECONDS) - .until(() -> inst.callOnInstance(() -> FailureDetector.instance.isAlive(DistributedTestSnitch.toCassandraInetAddressAndPort(downAddress)))); + .until(() -> inst.callOnInstance(() -> FailureDetector.instance.isAlive(TestEndpointCache.toCassandraInetAddressAndPort(downAddress)))); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java index ffc7f96bba..538bf4f491 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java @@ -25,11 +25,7 @@ import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.harry.sut.TokenPlacementModel; -import org.apache.cassandra.locator.IEndpointSnitch; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; @@ -50,23 +46,6 @@ public class CMSTestBase static { DatabaseDescriptor.daemonInitialization(); - DatabaseDescriptor.setEndpointSnitch(new IEndpointSnitch() - { - public String getRack(InetAddressAndPort endpoint) - { - ClusterMetadata metadata = ClusterMetadata.current(); - return metadata.directory.location(metadata.directory.peerId(endpoint)).rack; - } - public String getDatacenter(InetAddressAndPort endpoint) - { - ClusterMetadata metadata = ClusterMetadata.current(); - return metadata.directory.location(metadata.directory.peerId(endpoint)).datacenter; - } - public > C sortedByProximity(InetAddressAndPort address, C addresses) {return null;} - public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) {return 0;} - public void gossiperStarting() {} - public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) {return false;} - }); DatabaseDescriptor.setDefaultKeyspaceRF(1); Guardrails.instance.setMinimumReplicationFactorThreshold(1, 1); @@ -107,7 +86,7 @@ public class CMSTestBase ClusterMetadataService.setInstance(service); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); service.commit(new Initialize(ClusterMetadata.current()) { public Result execute(ClusterMetadata prev) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java index b27952d284..3c822864c5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java @@ -60,6 +60,7 @@ import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Commit; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; +import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.membership.Directory; @@ -135,7 +136,7 @@ public class ClusterMetadataTestHelper Commit.Replicator.NO_OP, true); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); QueryProcessor.registerStatementInvalidatingListener(); service.mark(); return service; @@ -281,11 +282,19 @@ public class ClusterMetadataTestHelper return register(addr(nodeIdx), dc, rack); } + public static NodeId register(InetAddressAndPort endpoint, Location location) + { + return register(endpoint, location.datacenter, location.rack); + } + public static NodeId register(InetAddressAndPort endpoint, String dc, String rack) { try { - return commit(new Register(addr(endpoint), new Location(dc, rack), NodeVersion.CURRENT)).directory.peerId(endpoint); + NodeId id = commit(new Register(addr(endpoint), new Location(dc, rack), NodeVersion.CURRENT)).directory.peerId(endpoint); + if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) + RegistrationStatus.instance.onRegistration(); + return id; } catch (Throwable e) { @@ -788,6 +797,11 @@ public class ClusterMetadataTestHelper addEndpoint(endpoint, tokens, "dc1", "rack1"); } + public static void addEndpoint(InetAddressAndPort endpoint, Collection tokens, Location location) + { + addEndpoint(endpoint, tokens, location.datacenter, location.rack); + } + public static void addEndpoint(InetAddressAndPort endpoint, Token t, Location location) { addEndpoint(endpoint, Collections.singleton(t), location.datacenter, location.rack); @@ -804,6 +818,8 @@ public class ClusterMetadataTestHelper { Location l = new Location(dc, rack); commit(new Register(addr(endpoint), l, NodeVersion.CURRENT)); + if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) + RegistrationStatus.instance.onRegistration(); lazyJoin(endpoint, new HashSet<>(t)).prepareJoin() .startJoin() .midJoin() diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java index 64c4e629e3..af9f1806bf 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java @@ -664,7 +664,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase true); ClusterMetadataService.setInstance(service); log.readyUnchecked(); - log.bootstrap(cms.addr()); + log.unsafeBootstrapForTesting(cms.addr()); service.commit(new Initialize(log.metadata())); service.commit(new Register(new NodeAddresses(cms.addr()), new Location(cms.dc(), cms.rack()), NodeVersion.CURRENT)); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java index 931ee320e5..7e2d1880a2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java @@ -83,65 +83,12 @@ import static org.apache.cassandra.harry.sut.TokenPlacementModel.nodeFactoryHuma public class MetadataChangeSimulationTest extends CMSTestBase { private static final Logger logger = LoggerFactory.getLogger(MetadataChangeSimulationTest.class); - private static final long seed; - private static final Random rng; static { - seed = System.nanoTime(); - logger.info("SEED: {}", seed); - rng = new Random(seed); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); } - - @Test - public void simulateNTS() throws Throwable - { - // TODO: right now, we pick a candidate only if there is enough rf to execute operation - // but the problem is that if we start multiple operations that would take us under rf, we will screw up the placements - // this was not happening before, and test is crafted now to disallow such states, but this is a bug. - // we should either forbid this, or allow it, but make it work. - for (int concurrency : new int[]{ 1, 3, 5 }) - { - for (int rf : new int[]{ 2, 3, 5 }) - { - for (int trans = 0; trans < rf; trans++) - { - simulate(50, 0, new NtsReplicationFactor(3, rf, trans), concurrency); - } - } - } - } - - @Test - public void simulateSimple() throws Throwable - { - for (int concurrency : new int[]{ 1, 3, 5 }) - { - for (int rf : new int[]{ 2, 3, 5 }) - { - for (int trans = 0; trans < rf; trans++) - { - simulate(50, 0, new SimpleReplicationFactor(rf, trans), concurrency); - } - } - } - } - - @Test - public void simulateSimpleOneTransient() throws Throwable - { - DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); - simulate(50, 0, new SimpleReplicationFactor(5, 2), 1); - } - - @Test - public void simulateSimpleOneNonTransient() throws Throwable - { - simulate(50, 0, new SimpleReplicationFactor(3), 1); - } - @Test public void testMoveReal() throws Throwable { @@ -395,7 +342,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase } } - public void simulate(int toBootstrap, int minSteps, ReplicationFactor rf, int concurrency) throws Throwable + public static void simulate(long seed, Random rng, int toBootstrap, int minSteps, ReplicationFactor rf, int concurrency) throws Throwable { logger.info("RUNNING SIMULATION WITH SEED {}. TO BOOTSTRAP: {}, RF: {}, CONCURRENCY: {}", seed, toBootstrap, rf, concurrency); long startTime = System.currentTimeMillis(); @@ -533,7 +480,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase } } - public void simulateBounces(ReplicationFactor rf, CMSPlacementStrategy CMSConfigurationStrategy, Random random) throws Throwable + public static void simulateBounces(ReplicationFactor rf, CMSPlacementStrategy CMSConfigurationStrategy, Random random) throws Throwable { try(CMSSut sut = new CMSSut(AtomicLongBackedProcessor::new, false, rf)) { @@ -631,7 +578,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase return pair(newState, node); } - private ModelChecker.Pair registerNewNode(ModelState state, CMSSut sut, int tokenIdx, int dcIdx, int rackIdx) + private static ModelChecker.Pair registerNewNode(ModelState state, CMSSut sut, int tokenIdx, int dcIdx, int rackIdx) { ModelState newState = state.transformer().incrementUniqueNodes().transform(); Node node = state.nodeFactory.make(newState.uniqueNodes, dcIdx, rackIdx).withToken(tokenIdx); @@ -639,7 +586,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase return pair(newState, node); } - private ModelChecker.Pair registerNewNodeWithToken(ModelState state, CMSSut sut, long token, int dcIdx, int rackIdx) + private static ModelChecker.Pair registerNewNodeWithToken(ModelState state, CMSSut sut, long token, int dcIdx, int rackIdx) { ModelState newState = state.transformer().incrementUniqueNodes().transform(); Node node = state.nodeFactory.make(newState.uniqueNodes, dcIdx, rackIdx).overrideToken(token); @@ -647,17 +594,17 @@ public class MetadataChangeSimulationTest extends CMSTestBase return pair(newState, node); } - private Node getRemovalCandidate(ModelState state, EntropySource entropySource) + private static Node getRemovalCandidate(ModelState state, EntropySource entropySource) { return getCandidate(state, entropySource); } - private Node getMoveCandidate(ModelState state, EntropySource entropySource) + private static Node getMoveCandidate(ModelState state, EntropySource entropySource) { return getCandidate(state, entropySource); } - private Node getCandidate(ModelState modelState, EntropySource entropySource) + private static Node getCandidate(ModelState modelState, EntropySource entropySource) { List dcs = new ArrayList<>(modelState.simulatedPlacements.rf.asMap().keySet()); while (!dcs.isEmpty()) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/NTSSimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/NTSSimulationTest.java new file mode 100644 index 0000000000..6e055b257c --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/log/NTSSimulationTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.log; + +import java.util.Random; + +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; + +import static org.apache.cassandra.distributed.test.log.MetadataChangeSimulationTest.simulate; +import static org.apache.cassandra.harry.sut.TokenPlacementModel.NtsReplicationFactor; + +public class NTSSimulationTest extends CMSTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(NTSSimulationTest.class); + private long seed; + private Random rng; + static + { + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + } + + @Before + public void setup() + { + seed = System.nanoTime(); + logger.info("SEED: {}", seed); + rng = new Random(seed); + } + + @Test + public void simulateNTS() throws Throwable + { + // TODO: right now, we pick a candidate only if there is enough rf to execute operation + // but the problem is that if we start multiple operations that would take us under rf, we will screw up the placements + // this was not happening before, and test is crafted now to disallow such states, but this is a bug. + // we should either forbid this, or allow it, but make it work. + for (int concurrency : new int[]{ 1, 3, 5 }) + { + for (int rf : new int[]{ 2, 3, 5 }) + { + for (int trans = 0; trans < rf; trans++) + { + simulate(seed, rng, 50, 0, new NtsReplicationFactor(3, rf, trans), concurrency); + } + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SimpleStrategySimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/SimpleStrategySimulationTest.java new file mode 100644 index 0000000000..0aaf8737eb --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SimpleStrategySimulationTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.log; + +import java.util.Random; + +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; + +import static org.apache.cassandra.distributed.test.log.MetadataChangeSimulationTest.simulate; +import static org.apache.cassandra.harry.sut.TokenPlacementModel.SimpleReplicationFactor; + +public class SimpleStrategySimulationTest extends CMSTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(SimpleStrategySimulationTest.class); + private long seed; + private Random rng; + static + { + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + } + + @Before + public void setup() + { + seed = System.nanoTime(); + logger.info("SEED: {}", seed); + rng = new Random(seed); + } + + + @Test + public void simulateSimple() throws Throwable + { + for (int concurrency : new int[]{ 1, 3, 5 }) + { + for (int rf : new int[]{ 2, 3, 5 }) + { + for (int trans = 0; trans < rf; trans++) + { + simulate(seed, rng, 50, 0, new SimpleReplicationFactor(rf, trans), concurrency); + } + } + } + } + + @Test + public void simulateSimpleOneTransient() throws Throwable + { + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + simulate(seed, rng, 50, 0, new SimpleReplicationFactor(5, 2), 1); + } + + @Test + public void simulateSimpleOneNonTransient() throws Throwable + { + simulate(seed, rng, 50, 0, new SimpleReplicationFactor(3), 1); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java index 821b6a1cef..69ccafdd45 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java +++ b/test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java @@ -109,7 +109,7 @@ public abstract class DecommissionAvoidTimeouts extends TestBaseImpl pending.call(); unpauseCommits(cluster.get(1)); - cluster.forEach(i -> i.runOnInstance(() -> ((DynamicEndpointSnitch) DatabaseDescriptor.getEndpointSnitch()).updateScores())); + cluster.forEach(i -> i.runOnInstance(() -> ((DynamicEndpointSnitch) DatabaseDescriptor.getNodeProximity()).updateScores())); cluster.filters().verbs(Verb.GOSSIP_DIGEST_SYN.id).drop(); nodetool.join(); diff --git a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java index bc7eb3b17b..1f3edcf446 100644 --- a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java +++ b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java @@ -54,8 +54,8 @@ public class DynamicEndpointSnitchLongTest // do this because SS needs to be initialized before DES can work properly. StorageService.instance.unsafeInitialize(); - SimpleSnitch ss = new SimpleSnitch(); - DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode())); + NodeProximity proximity = new NoOpProximity(); + DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(proximity, String.valueOf(proximity.hashCode())); InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); EndpointsForRange.Builder replicasBuilder = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE); diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSnitch.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSnitch.java index 4000946a94..55fe73c301 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSnitch.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSnitch.java @@ -104,6 +104,8 @@ public class SimulatedSnitch extends NodeLookup if (prev != null) prev.accept(config); config.set("endpoint_snitch", SimulatedSnitch.Instance.class.getName()) + .set("initial_location_provider", null) + .set("node_proximity", null) .set("dynamic_snitch", false); }); } diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java index 29e1a84a1e..596312ff0b 100644 --- a/test/unit/org/apache/cassandra/ServerTestUtils.java +++ b/test/unit/org/apache/cassandra/ServerTestUtils.java @@ -44,9 +44,9 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.big.BigTableReader; import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport; import org.apache.cassandra.io.util.File; -import org.apache.cassandra.locator.AbstractEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.BaseProximity; import org.apache.cassandra.security.ThreadAwareSecurityManager; import org.apache.cassandra.service.EmbeddedCassandraService; import org.apache.cassandra.tcm.AtomicLongBackedProcessor; @@ -56,9 +56,11 @@ import org.apache.cassandra.tcm.Commit; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; import org.apache.cassandra.tcm.Processor; +import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.PlacementProvider; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; @@ -98,23 +100,8 @@ public final class ServerTestUtils public static void initSnitch() { // Register an EndpointSnitch which returns fixed values for test. - DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() + DatabaseDescriptor.setNodeProximity(new BaseProximity() { - @Override - public String getRack(InetAddressAndPort endpoint) - { - return RACK1; - } - - @Override - public String getDatacenter(InetAddressAndPort endpoint) - { - if (remoteAddrs.contains(endpoint)) - return DATA_CENTER_REMOTE; - - return DATA_CENTER; - } - @Override public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { @@ -135,6 +122,7 @@ public final class ServerTestUtils tokens, ClusterMetadataService.instance().placementProvider())); SystemKeyspace.setLocalHostId(nodeId.toUUID()); + RegistrationStatus.instance.onRegistration(); return nodeId; } @@ -274,6 +262,7 @@ public final class ServerTestUtils Function processorFactory = AtomicLongBackedProcessor::new; IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Location location = DatabaseDescriptor.getLocator().local(); boolean addListeners = true; ClusterMetadata initial = new ClusterMetadata(partitioner); if (!Keyspace.isInitialized()) @@ -293,7 +282,7 @@ public final class ServerTestUtils ClusterMetadataService.setInstance(service); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.bootstrap(FBUtilities.getBroadcastAddressAndPort(), location.datacenter); service.commit(new Initialize(ClusterMetadata.current())); QueryProcessor.registerStatementInvalidatingListener(); service.mark(); @@ -328,7 +317,7 @@ public final class ServerTestUtils ClusterMetadataService.setInstance(cms); ((SystemKeyspaceStorage)LogStorage.SystemKeyspace).truncate(); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); cms.mark(); } diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java index c5041e6e70..4b3e048134 100644 --- a/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java +++ b/test/unit/org/apache/cassandra/batchlog/BatchlogEndpointFilterTest.java @@ -30,11 +30,9 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; import org.junit.After; import org.junit.Assert; @@ -44,14 +42,16 @@ import org.junit.Test; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.locator.DynamicEndpointSnitch; -import org.apache.cassandra.locator.GossipingPropertyFileSnitch; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.NoOpProximity; +import org.apache.cassandra.locator.NodeProximity; import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.StubClusterMetadataService; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.FBUtilities; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -275,7 +275,8 @@ public class BatchlogEndpointFilterTest filterBatchlogEndpointsRandomForTests(false, endpoints); } - private Collection filterBatchlogEndpointsDynamicForTests(Multimap endpoints) { + private Collection filterBatchlogEndpointsDynamicForTests(Multimap endpoints) + { return ReplicaPlans.filterBatchlogEndpointsDynamic(false, LOCAL, endpoints, x -> true); } @@ -972,57 +973,42 @@ public class BatchlogEndpointFilterTest assert !racks.isEmpty(); assert racks.size() <= 10; assert racks.size() == nodesPerRack.length; - + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(ClusterMetadataTestHelper.instanceForTest()); ImmutableMultimap.Builder builder = ImmutableMultimap.builder(); for (int r = 0; r < racks.size(); r++) { String rack = racks.get(r); for (int n = 0; n < nodesPerRack[r]; n++) + { builder.put(rack, endpointAddress(r, n)); + ClusterMetadataTestHelper.register(endpointAddress(r, n), new Location("dc1", rack)); + } } ImmutableMultimap endpoints = builder.build(); - reconfigure(batchlogEndpointStrategy, dynamicSnitch, endpoints); + reconfigure(batchlogEndpointStrategy, dynamicSnitch); return endpoints; } private void reconfigure(Config.BatchlogEndpointStrategy batchlogEndpointStrategy, - boolean dynamicSnitch, - Multimap endpoints) + boolean dynamicSnitch) { DatabaseDescriptor.setBatchlogEndpointStrategy(batchlogEndpointStrategy); - if (DatabaseDescriptor.getEndpointSnitch() instanceof DynamicEndpointSnitch) - ((DynamicEndpointSnitch) DatabaseDescriptor.getEndpointSnitch()).close(); + if (DatabaseDescriptor.getNodeProximity() instanceof DynamicEndpointSnitch) + ((DynamicEndpointSnitch) DatabaseDescriptor.getNodeProximity()).close(); - Multimap endpointRacks = Multimaps.invertFrom(endpoints, ArrayListMultimap.create()); - GossipingPropertyFileSnitch gpfs = new GossipingPropertyFileSnitch() - { - @Override - public String getDatacenter(InetAddressAndPort endpoint) - { - return "dc1"; - } - - @Override - public String getRack(InetAddressAndPort endpoint) - { - return endpointRacks.get(endpoint).iterator().next(); - } - }; - IEndpointSnitch snitch; + NodeProximity nodeProximity = new NoOpProximity(); if (dynamicSnitch) - snitch = dsnitch = new DynamicEndpointSnitch(gpfs, String.valueOf(gpfs.hashCode())); + nodeProximity = dsnitch = new DynamicEndpointSnitch(nodeProximity, String.valueOf(nodeProximity.hashCode())); else - { dsnitch = null; - snitch = gpfs; - } DatabaseDescriptor.setDynamicBadnessThreshold(0); - DatabaseDescriptor.setEndpointSnitch(snitch); + DatabaseDescriptor.setNodeProximity(nodeProximity); DatabaseDescriptor.setBatchlogEndpointStrategy(batchlogEndpointStrategy); } @@ -1046,7 +1032,7 @@ public class BatchlogEndpointFilterTest private String configToString(Multimap endpoints) { return "strategy:" + DatabaseDescriptor.getBatchlogEndpointStrategy() - + " snitch:" + DatabaseDescriptor.getEndpointSnitch().getClass().getSimpleName() + + " proximity:" + DatabaseDescriptor.getNodeProximity().getClass().getSimpleName() + " nodes-per-rack: " + endpoints.asMap().entrySet().stream() .map(e -> e.getKey() + '=' + e.getValue().size()) .collect(Collectors.joining()); diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 96b164a322..89af53b175 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -206,6 +206,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.exceptions.TransportException", "org.apache.cassandra.fql.FullQueryLogger", "org.apache.cassandra.fql.FullQueryLoggerOptions", + "org.apache.cassandra.gms.IEndpointStateChangeSubscriber", "org.apache.cassandra.gms.IFailureDetector", "org.apache.cassandra.io.FSError", "org.apache.cassandra.io.FSWriteError", @@ -269,15 +270,21 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy", "org.apache.cassandra.locator.IEndpointSnitch", "org.apache.cassandra.locator.InetAddressAndPort", + "org.apache.cassandra.locator.Locator", + "org.apache.cassandra.locator.NodeAddressConfig", + "org.apache.cassandra.locator.NodeProximity", + "org.apache.cassandra.locator.InitialLocationProvider", "org.apache.cassandra.locator.Replica", "org.apache.cassandra.locator.ReplicaCollection", "org.apache.cassandra.locator.SeedProvider", "org.apache.cassandra.locator.SimpleSeedProvider", + "org.apache.cassandra.locator.SnitchAdapter", "org.apache.cassandra.security.EncryptionContext", "org.apache.cassandra.security.ISslContextFactory", "org.apache.cassandra.security.SSLFactory", "org.apache.cassandra.service.CacheService$CacheType", "org.apache.cassandra.security.AbstractCryptoProvider", + "org.apache.cassandra.tcm.RegistrationStateCallbacks", "org.apache.cassandra.transport.ProtocolException", "org.apache.cassandra.utils.Closeable", "org.apache.cassandra.utils.CloseableIterator", diff --git a/test/unit/org/apache/cassandra/cql3/PagingTest.java b/test/unit/org/apache/cassandra/cql3/PagingTest.java index 919ef8722b..eddc7aebd5 100644 --- a/test/unit/org/apache/cassandra/cql3/PagingTest.java +++ b/test/unit/org/apache/cassandra/cql3/PagingTest.java @@ -31,11 +31,11 @@ import com.datastax.driver.core.SimpleStatement; import com.datastax.driver.core.Statement; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.locator.AbstractEndpointSnitch; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.NodeProximity; +import org.apache.cassandra.locator.BaseProximity; import org.apache.cassandra.service.EmbeddedCassandraService; import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG; @@ -102,23 +102,13 @@ public class PagingTest String createTableStatement = "CREATE TABLE IF NOT EXISTS " + table + " (id int, id2 int, id3 int, val text, PRIMARY KEY ((id, id2), id3));"; String dropTableStatement = "DROP TABLE IF EXISTS " + table + ';'; - // custom snitch to avoid merging ranges back together after StorageProxy#getRestrictedRanges splits them up - IEndpointSnitch snitch = new AbstractEndpointSnitch() + // custom proximity impl to avoid merging ranges back together after StorageProxy#getRestrictedRanges splits them up + NodeProximity proximity = new BaseProximity() { - private IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch(); + private final NodeProximity oldProximity = DatabaseDescriptor.getNodeProximity(); public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { - return oldSnitch.compareEndpoints(target, a1, a2); - } - - public String getRack(InetAddressAndPort endpoint) - { - return oldSnitch.getRack(endpoint); - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - return oldSnitch.getDatacenter(endpoint); + return oldProximity.compareEndpoints(target, a1, a2); } @Override @@ -127,9 +117,7 @@ public class PagingTest return false; } }; - DatabaseDescriptor.setEndpointSnitch(snitch); -// StorageService.instance.getTokenMetadata().clearUnsafe(); -// StorageService.instance.getTokenMetadata().updateNormalToken(new LongToken(5097162189738624638L), FBUtilities.getBroadcastAddressAndPort()); + DatabaseDescriptor.setNodeProximity(proximity); session.execute(createTableStatement); for (int i = 0; i < 110; i++) diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 1bef2d82cf..cd19414166 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -451,12 +451,12 @@ public class DescribeStatementTest extends CQLTester assertRowsNet(executeDescribeNet(describeKeyword + " CLUSTER"), row("Test Cluster", trimIfPresent(DatabaseDescriptor.getPartitionerName(), "org.apache.cassandra.dht."), - DatabaseDescriptor.getEndpointSnitch().getClass().getName())); + DatabaseDescriptor.getNodeProximity().getClass().getName())); assertRowsNet(executeDescribeNet("system_virtual_schema", describeKeyword + " CLUSTER"), row("Test Cluster", trimIfPresent(DatabaseDescriptor.getPartitionerName(), "org.apache.cassandra.dht."), - DatabaseDescriptor.getEndpointSnitch().getClass().getName())); + DatabaseDescriptor.getNodeProximity().getClass().getName())); } ClusterMetadata metadata = ClusterMetadata.current(); Token token = metadata.tokenMap.tokens().get(0); @@ -465,7 +465,7 @@ public class DescribeStatementTest extends CQLTester assertRowsNet(executeDescribeNet(KEYSPACE_PER_TEST, "DESCRIBE CLUSTER"), row("Test Cluster", trimIfPresent(DatabaseDescriptor.getPartitionerName(), "org.apache.cassandra.dht."), - DatabaseDescriptor.getEndpointSnitch().getClass().getName(), + DatabaseDescriptor.getNodeProximity().getClass().getName(), ImmutableMap.of(token.toString(), ImmutableList.of(addressAndPort.toString())))); } diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java index f4f94d1a30..d474fd1a19 100644 --- a/test/unit/org/apache/cassandra/db/CleanupTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTest.java @@ -39,7 +39,6 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.filter.RowFilter; @@ -48,7 +47,6 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -94,23 +92,6 @@ public class CleanupTest KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED1, true)); - - - DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch() - { - @Override - public String getRack(InetAddressAndPort endpoint) - { - return "RC1"; - } - - @Override - public String getDatacenter(InetAddressAndPort endpoint) - { - return "DC1"; - } - }); - SchemaLoader.createKeyspace(KEYSPACE2, KeyspaceParams.nts("DC1", 1), SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD2), diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java index 0550237b74..cdd7567fa4 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreClientModeTest.java @@ -30,7 +30,6 @@ import org.junit.rules.TemporaryFolder; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.locator.SimpleSnitch; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; @@ -66,11 +65,9 @@ public class ColumnFamilyStoreClientModeTest if (DatabaseDescriptor.getPartitioner() == null) DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); - DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch()); DatabaseDescriptor.getRawConfig().memtable_flush_writers = 1; DatabaseDescriptor.getRawConfig().local_system_data_file_directory = tempFolder.toString(); DatabaseDescriptor.getRawConfig().partitioner = "Murmur3Partitioner"; - DatabaseDescriptor.setLocalDataCenter("DC1"); DatabaseDescriptor.applyPartitioner(); } diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java index 2047210323..7eb8f9c578 100644 --- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java @@ -147,8 +147,8 @@ public class SystemKeyspaceTest assertEquals(FBUtilities.getReleaseVersionString(), row.getString("release_version")); assertEquals(QueryProcessor.CQL_VERSION.toString(), row.getString("cql_version")); assertEquals(String.valueOf(ProtocolVersion.CURRENT.asInt()), row.getString("native_protocol_version")); - assertEquals(DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter(), row.getString("data_center")); - assertEquals(DatabaseDescriptor.getEndpointSnitch().getLocalRack(), row.getString("rack")); + assertEquals(DatabaseDescriptor.getLocator().local().datacenter, row.getString("data_center")); + assertEquals(DatabaseDescriptor.getLocator().local().rack, row.getString("rack")); assertEquals(DatabaseDescriptor.getPartitioner().getClass().getName(), row.getString("partitioner")); assertEquals(FBUtilities.getJustBroadcastNativeAddress(), row.getInetAddress("rpc_address")); assertEquals(DatabaseDescriptor.getNativeTransportPort(), row.getInt("rpc_port")); diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java index 32fa3b234a..f2b47cbf6e 100644 --- a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java +++ b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java @@ -24,18 +24,15 @@ import java.util.Collection; import java.util.function.Consumer; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.runners.Parameterized; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.RegularAndStaticColumns; - import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.db.rows.CellPath; @@ -43,7 +40,6 @@ import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.locator.SimpleSnitch; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -93,18 +89,10 @@ public class ColumnFilterTest DatabaseDescriptor.daemonInitialization(); SchemaLoader.prepareServer(); DatabaseDescriptor.setSeedProvider(Arrays::asList); - DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch()); DatabaseDescriptor.setDefaultFailureDetector(); DatabaseDescriptor.setPartitionerUnsafe(new Murmur3Partitioner()); } - @Before - public void before() - { - Util.setUpgradeFromVersion("4.0"); - // todo; nothing in this test is actually version dependent anymore - } - // Select all @Test diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java index 04a7e33b3e..d35673a847 100644 --- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java +++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumReplicationFactorTest.java @@ -26,14 +26,10 @@ import java.util.stream.Collectors; import org.junit.After; import org.junit.Test; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; -import org.apache.cassandra.locator.AbstractEndpointSnitch; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.service.ClientWarn; import org.assertj.core.api.Assertions; @@ -113,21 +109,6 @@ public class GuardrailMaximumReplicationFactorTest extends ThresholdTester @Test public void testMultipleDatacenter() throws Throwable { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() - { - public static final String RACK1 = ServerTestUtils.RACK1; - - @Override - public String getRack(InetAddressAndPort endpoint) { return RACK1; } - - @Override - public String getDatacenter(InetAddressAndPort endpoint) { return "datacenter2"; } - - @Override - public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } - }); - List twoWarnings = Arrays.asList(format("The keyspace ks has a replication factor of 3, above the warning threshold of %s.", MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD), format("The keyspace ks has a replication factor of 3, above the warning threshold of %s.", MAXIMUM_REPLICATION_FACTOR_WARN_THRESHOLD)); @@ -152,7 +133,7 @@ public class GuardrailMaximumReplicationFactorTest extends ThresholdTester assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 2, 'datacenter2' : 5}", 5); assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 5, 'datacenter2' : 5}", 5); - DatabaseDescriptor.setEndpointSnitch(snitch); +// DatabaseDescriptor.setEndpointSnitch(snitch); } @Test diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java index 173f70698b..9b376e2f0e 100644 --- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java +++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumReplicationFactorTest.java @@ -27,14 +27,11 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.locator.AbstractEndpointSnitch; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.service.ClientWarn; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.transformations.Register; import org.assertj.core.api.Assertions; @@ -153,26 +150,11 @@ public class GuardrailMinimumReplicationFactorTest extends ThresholdTester @Test public void testMultipleDatacenter() throws Throwable { - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() - { - public static final String RACK1 = ServerTestUtils.RACK1; - - @Override - public String getRack(InetAddressAndPort endpoint) { return RACK1; } - - @Override - public String getDatacenter(InetAddressAndPort endpoint) { return "datacenter2"; } - - @Override - public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } - }); - List twoWarnings = Arrays.asList(format("The keyspace %s has a replication factor of 2, below the warning threshold of %d.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD), format("The keyspace %s has a replication factor of 2, below the warning threshold of %d.", KS, MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD)); InetAddressAndPort ep = InetAddressAndPort.getByName("127.0.0.255"); - Register.register(new NodeAddresses(ep)); + Register.register(new NodeAddresses(ep), new Location("datacenter2", "rack1")); guardrails().setMinimumReplicationFactorThreshold(MINIMUM_REPLICATION_FACTOR_WARN_THRESHOLD, MINIMUM_REPLICATION_FACTOR_FAIL_THRESHOLD); assertValid("CREATE KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 4 }"); execute("DROP KEYSPACE IF EXISTS ks"); @@ -192,7 +174,6 @@ public class GuardrailMinimumReplicationFactorTest extends ThresholdTester assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 4, 'datacenter2' : 1 }", 1); assertFails("ALTER KEYSPACE ks WITH replication = { 'class' : 'NetworkTopologyStrategy', 'datacenter1': 1, 'datacenter2' : 1 }", 1); - DatabaseDescriptor.setEndpointSnitch(snitch); execute("DROP KEYSPACE IF EXISTS ks1"); } diff --git a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java index 18463c7122..2e6870e6a6 100644 --- a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java +++ b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java @@ -54,7 +54,6 @@ public class ViewUtilsTest DatabaseDescriptor.setPartitionerUnsafe(OrderPreservingPartitioner.instance); ServerTestUtils.cleanupAndLeaveDirs(); Keyspace.setInitialized(); - } @Before diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index 1407cb8b97..f82a51fc58 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -129,7 +129,7 @@ public class BootStrapperTest RangeStreamer s = new RangeStreamer(metadata, StreamOperation.BOOTSTRAP, true, - DatabaseDescriptor.getEndpointSnitch(), + DatabaseDescriptor.getNodeProximity(), new StreamStateStore(), mockFailureDetector, false, diff --git a/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java b/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java index bb02f40718..12015249ba 100644 --- a/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java +++ b/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java @@ -32,39 +32,42 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.tcm.membership.Location; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class RangeFetchMapCalculatorTest { + private static Locator locator; @BeforeClass - public static void setupUpSnitch() + public static void setup() { DatabaseDescriptor.daemonInitialization(); DatabaseDescriptor.setPartitionerUnsafe(RandomPartitioner.instance); - DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch() + locator = new Locator(null, null, null, null) { - //Odd IPs are in DC1 and Even are in DC2. Endpoints upto .14 will have unique racks and - // then will be same for a set of three. + Location local = new Location("DC1", "RAC1"); + Location remote = new Location("DC2", "RAC1"); @Override - public String getRack(InetAddressAndPort endpoint) + public Location local() { - return "RAC1"; + return local; } @Override - public String getDatacenter(InetAddressAndPort endpoint) + public Location location(InetAddressAndPort endpoint) { + //Odd IPs are in DC1 and Even are in DC2. if (getIPLastPart(endpoint) <= 50) - return DatabaseDescriptor.getLocalDataCenter(); + return local; else if (getIPLastPart(endpoint) % 2 == 0) - return DatabaseDescriptor.getLocalDataCenter(); + return local; else - return DatabaseDescriptor.getLocalDataCenter() + "Remote"; + return remote; } private int getIPLastPart(InetAddressAndPort endpoint) @@ -73,7 +76,7 @@ public class RangeFetchMapCalculatorTest int index = str.lastIndexOf("."); return Integer.parseInt(str.substring(index + 1).trim()); } - }); + }; } @Test @@ -86,7 +89,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.4"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.5"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -103,7 +106,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.7", "127.0.0.8"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.9", "127.0.0.10"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -118,7 +121,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2", "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3", "127.0.0.4"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -136,7 +139,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.3", "127.0.0.2"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -158,7 +161,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.1", "127.0.0.2"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -178,7 +181,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.1"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); //All ranges map to local host so we will not stream anything. assertTrue(map.isEmpty()); @@ -216,7 +219,7 @@ public class RangeFetchMapCalculatorTest } }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(filter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(filter), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -249,7 +252,7 @@ public class RangeFetchMapCalculatorTest } }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(allDeadFilter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(allDeadFilter), "Test", locator); calculator.getRangeFetchMap(); } @@ -261,7 +264,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1", "127.0.0.3", "127.0.0.57"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59", "127.0.0.61"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), new ArrayList<>(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), new ArrayList<>(), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); Assert.assertEquals(2, map.asMap().size()); @@ -303,7 +306,7 @@ public class RangeFetchMapCalculatorTest } }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(localHostFilter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Arrays.asList(localHostFilter), "Test", locator); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); Assert.assertEquals(3, map.asMap().size()); @@ -324,7 +327,7 @@ public class RangeFetchMapCalculatorTest addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59"); // and a trivial one: addTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.emptyList(), "Test", locator); Multimap> optMap = calculator.getRangeFetchMapForNonTrivialRanges(); Multimap> trivialMap = calculator.getRangeFetchMapForTrivialRanges(optMap); assertTrue(trivialMap.get(InetAddressAndPort.getByName("127.0.0.3")).contains(generateTrivialRange(1,10)) ^ @@ -364,7 +367,7 @@ public class RangeFetchMapCalculatorTest return "Not 127.0.0.3"; } }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.singleton(filter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.build(), Collections.singleton(filter), "Test", locator); Multimap> optMap = calculator.getRangeFetchMapForNonTrivialRanges(); Multimap> trivialMap = calculator.getRangeFetchMapForTrivialRanges(optMap); @@ -378,7 +381,7 @@ public class RangeFetchMapCalculatorTest addTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.1"); addTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1", "127.0.0.2"); EndpointsByRange ebr = rangesWithSources.build(); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(ebr, Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(ebr, Collections.emptyList(), "Test", locator); RangeStreamer.validateRangeFetchMap(ebr, calculator.getRangeFetchMap(), "Test"); } diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java index f0e0cda24d..6b4ef40465 100644 --- a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java +++ b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.dht.tokenallocator; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Random; @@ -33,6 +34,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; @@ -40,14 +42,15 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractReplicationStrategy; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.RackInferringSnitch; +import org.apache.cassandra.locator.SimpleLocationProvider; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.StubClusterMetadataService; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -69,7 +72,7 @@ public class TokenAllocationTest public void before() throws ConfigurationException { ClusterMetadataService.setInstance(ClusterMetadataTestHelper.syncInstanceForTest()); - ClusterMetadataService.instance().log().bootstrap(FBUtilities.getBroadcastAddressAndPort()); + ClusterMetadataService.instance().log().unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); } @Before @@ -86,7 +89,7 @@ public class TokenAllocationTest private static TokenAllocation createForTest(ClusterMetadata metadata, int replicas, int numTokens) { - return TokenAllocation.create(DatabaseDescriptor.getEndpointSnitch(), metadata, replicas, numTokens); + return TokenAllocation.create(metadata.locator.local().datacenter, metadata, replicas, numTokens); } @Test @@ -169,88 +172,72 @@ public class TokenAllocationTest public void testAllocateTokensNetworkStrategy(int rackCount, int replicas) throws UnknownHostException { - IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch(); - try - { - DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch()); - int vn = 16; - String ks = "TokenAllocationTestNTSKeyspace" + rackCount + replicas; - String dc = "1"; - String otherDc = "15"; - KeyspaceMetadata keyspace = KeyspaceMetadata.create(ks, KeyspaceParams.nts(dc, replicas, otherDc, "15")); + int vn = 16; + String ks = "TokenAllocationTestNTSKeyspace" + rackCount + replicas; + String dc = "1"; + String otherDc = "15"; + KeyspaceMetadata keyspace = KeyspaceMetadata.create(ks, KeyspaceParams.nts(dc, replicas, otherDc, "15")); - // register these 2 nodes so that the DCs exist, otherwise the CREATE KEYSPACE will be rejected - // but don't join them, we don't assign any tokens to these nodes - ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.1.0.99"), dc, Integer.toString(0)); - ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.15.0.99"), otherDc, Integer.toString(0)); - ClusterMetadataTestHelper.addOrUpdateKeyspace(keyspace); + // register these 2 nodes so that the DCs exist, otherwise the CREATE KEYSPACE will be rejected + // but don't join them, we don't assign any tokens to these nodes + ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.1.0.99"), dc, Integer.toString(0)); + ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.15.0.99"), otherDc, Integer.toString(0)); + ClusterMetadataTestHelper.addOrUpdateKeyspace(keyspace); - for (int i = 0; i < rackCount; ++i) - generateFakeEndpoints(10, vn, dc, Integer.toString(i)); - InetAddressAndPort addr = InetAddressAndPort.getByName("127." + dc + ".0.99"); - allocateTokensForKeyspace(vn, ks, ClusterMetadata.current(), addr); - // Note: Not matching replication factor in second datacentre, but this should not affect us. - } finally { - DatabaseDescriptor.setEndpointSnitch(oldSnitch); - } + for (int i = 0; i < rackCount; ++i) + generateFakeEndpoints(10, vn, dc, Integer.toString(i)); + InetAddressAndPort addr = InetAddressAndPort.getByName("127." + dc + ".0.99"); + allocateTokensForKeyspace(vn, ks, ClusterMetadata.current(), addr); + // Note: Not matching replication factor in second datacentre, but this should not affect us. } @Test public void testAllocateTokensRfEqRacks() throws UnknownHostException { - IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch(); - try + int vn = 8; + int replicas = 3; + int rackCount = replicas; + String ks = "TokenAllocationTestNTSKeyspaceRfEqRack"; + String dc = "1"; + String otherDc = "15"; + KeyspaceMetadata keyspace = KeyspaceMetadata.create(ks, KeyspaceParams.nts(dc, replicas, otherDc, "15")); + + // register these 2 nodes so that the DCs exist, otherwise the CREATE KEYSPACE will be rejected + // but don't join them, we don't assign any tokens to these nodes + ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.1.0.255"), dc, Integer.toString(0)); + ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.15.0.255"), otherDc, Integer.toString(0)); + ClusterMetadataTestHelper.addOrUpdateKeyspace(keyspace); + + int base = 5; + for (int i = 0; i < rackCount; ++i) + generateFakeEndpoints(base << i, vn, dc, Integer.toString(i)); // unbalanced racks + + int cnt = 5; + for (int i = 0; i < cnt; ++i) { - DatabaseDescriptor.setEndpointSnitch(new RackInferringSnitch()); - int vn = 8; - int replicas = 3; - int rackCount = replicas; - String ks = "TokenAllocationTestNTSKeyspaceRfEqRack"; - String dc = "1"; - String otherDc = "15"; - KeyspaceMetadata keyspace = KeyspaceMetadata.create(ks, KeyspaceParams.nts(dc, replicas, otherDc, "15")); - - // register these 2 nodes so that the DCs exist, otherwise the CREATE KEYSPACE will be rejected - // but don't join them, we don't assign any tokens to these nodes - ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.1.0.255"), dc, Integer.toString(0)); - ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.15.0.255"), otherDc, Integer.toString(0)); - ClusterMetadataTestHelper.addOrUpdateKeyspace(keyspace); - - int base = 5; - for (int i = 0; i < rackCount; ++i) - generateFakeEndpoints(base << i, vn, dc, Integer.toString(i)); // unbalanced racks - - int cnt = 5; - for (int i = 0; i < cnt; ++i) - { - InetAddressAndPort endpoint = InetAddressAndPort.getByName("127." + dc + ".0." + (99 + i)); - Collection tokens = allocateTokensForKeyspace(vn, ks, ClusterMetadata.current(), endpoint); - - ClusterMetadataTestHelper.register(endpoint, dc, Integer.toString(0)); - ClusterMetadataTestHelper.join(endpoint, new HashSet<>(tokens)); - } - - double target = 1.0 / (base + cnt); - double permittedOver = 1.0 / (2 * vn + 1) + 0.01; - - Map ownership = StorageService.instance.effectiveOwnership(ks); - boolean failed = false; - for (Map.Entry o : ownership.entrySet()) - { - int rack = o.getKey().getAddress()[2]; - if (rack != 0) - continue; - - System.out.format("Node %s owns %f ratio to optimal %.2f\n", o.getKey(), o.getValue(), o.getValue() / target); - if (o.getValue()/target > 1 + permittedOver) - failed = true; - } - Assert.assertFalse(String.format("One of the nodes in the rack has over %.2f%% overutilization.", permittedOver * 100), failed); - } finally { - DatabaseDescriptor.setEndpointSnitch(oldSnitch); + InetAddressAndPort endpoint = InetAddressAndPort.getByName("127." + dc + ".0." + (99 + i)); + ClusterMetadataTestHelper.register(endpoint, dc, Integer.toString(0)); + Collection tokens = allocateTokensForKeyspace(vn, ks, ClusterMetadata.current(), endpoint); + ClusterMetadataTestHelper.join(endpoint, new HashSet<>(tokens)); } - } + double target = 1.0 / (base + cnt); + double permittedOver = 1.0 / (2 * vn + 1) + 0.01; + + Map ownership = StorageService.instance.effectiveOwnership(ks); + boolean failed = false; + for (Map.Entry o : ownership.entrySet()) + { + int rack = o.getKey().getAddress()[2]; + if (rack != 0) + continue; + + System.out.format("Node %s owns %f ratio to optimal %.2f\n", o.getKey(), o.getValue(), o.getValue() / target); + if (o.getValue() / target > 1 + permittedOver) + failed = true; + } + Assert.assertFalse(String.format("One of the nodes in the rack has over %.2f%% overutilization.", permittedOver * 100), failed); + } /** * TODO: This scenario isn't supported very well. Investigate a multi-keyspace version of the algorithm. @@ -260,7 +247,19 @@ public class TokenAllocationTest { final int TOKENS = 16; - ClusterMetadata metadata = generateFakeEndpoints(10, TOKENS); + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools()); + generateFakeEndpoints(10, TOKENS); + // pre-register the nodes which we are going to allocate tokens to this is necessary as + // Location info is obtained by the allocator from cluster metadata, not the snitch + Map endpointToIdMap = new HashMap<>(10); + for (int i=11; i<=20; ++i) + { + InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0." + (i + 1)); + NodeId id = ClusterMetadataTestHelper.register(endpoint, SimpleLocationProvider.LOCATION); + endpointToIdMap.put(endpoint, id); + } + ClusterMetadata metadata = ClusterMetadata.current(); TokenAllocation rf2Allocator = createForTest(metadata, 2, TOKENS); TokenAllocation rf3Allocator = createForTest(metadata, 3, TOKENS); @@ -271,12 +270,13 @@ public class TokenAllocationTest TokenAllocation current = rf3Allocator; TokenAllocation next = rf2Allocator; - for (int i=11; i<=20; ++i) + for (Map.Entry entry : endpointToIdMap.entrySet()) { - InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0." + (i + 1)); + InetAddressAndPort endpoint = entry.getKey(); + NodeId id = entry.getValue(); Set tokens = new HashSet<>(current.allocate(endpoint)); - ClusterMetadataTestHelper.register(endpoint, "datacenter" + 1, "rack" + 1); - ClusterMetadataTestHelper.join(endpoint, tokens); + // Update tokens on next to verify ownership calculation below + next.updateTokensForNode(id, tokens); TokenAllocation tmp = current; current = next; next = tmp; @@ -297,16 +297,33 @@ public class TokenAllocationTest } } - private ClusterMetadata generateFakeEndpoints(int numOldNodes, int numVNodes) throws UnknownHostException + private ClusterMetadata generateFakeEndpoints(int nodes, int vnodes) throws UnknownHostException { - return generateFakeEndpoints(numOldNodes, numVNodes, "0", "0"); + // This is used by tests which previously relied on the ServerTestUtils snitch impl which has + // 127.0.0.4 in datacenter2 and every other node in datacenter1 + IPartitioner p = ClusterMetadata.current().tokenMap.partitioner(); + for (int i = 1; i <= nodes; i++) + { + // leave .1 for myEndpoint + InetAddressAndPort addr = InetAddressAndPort.getByName("127.0.0." + (i + 1)); + String dc = (i + 1 == 4) ? ServerTestUtils.DATA_CENTER_REMOTE : ServerTestUtils.DATA_CENTER; + ClusterMetadataTestHelper.register(addr, dc, ServerTestUtils.RACK1); + Set tokens = new HashSet<>(vnodes); + for (int j = 0; j < vnodes; ++j) + tokens.add(p.getRandomToken(rand)); + ClusterMetadataTestHelper.join(addr, tokens); + } + // Register (but don't join/allocate tokens for) 127.0.0.1) + ClusterMetadataTestHelper.register(FBUtilities.getBroadcastAddressAndPort(), + ServerTestUtils.DATA_CENTER, + ServerTestUtils.RACK1); + return ClusterMetadata.current(); } private ClusterMetadata generateFakeEndpoints(int nodes, int vnodes, String dc, String rack) throws UnknownHostException { System.out.printf("Adding %d nodes to dc=%s, rack=%s.%n", nodes, dc, rack); - ClusterMetadata metadata = ClusterMetadata.current(); - IPartitioner p = metadata.tokenMap.partitioner(); + IPartitioner p = ClusterMetadata.current().tokenMap.partitioner(); for (int i = 1; i <= nodes; i++) { @@ -318,6 +335,6 @@ public class TokenAllocationTest tokens.add(p.getRandomToken(rand)); ClusterMetadataTestHelper.join(addr, tokens); } - return metadata; + return ClusterMetadata.current(); } } diff --git a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java index 6ca684c768..09591f291b 100644 --- a/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/AlibabaCloudSnitchTest.java @@ -19,20 +19,19 @@ package org.apache.cassandra.locator; import java.io.IOException; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION; import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; -import static org.apache.cassandra.locator.AlibabaCloudSnitch.DEFAULT_METADATA_SERVICE_URL; +import static org.apache.cassandra.locator.AlibabaCloudLocationProvider.DEFAULT_METADATA_SERVICE_URL; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; @@ -45,11 +44,16 @@ public class AlibabaCloudSnitchTest @BeforeClass public static void setup() throws Exception { - GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true); DatabaseDescriptor.daemonInitialization(); ClusterMetadataTestHelper.setInstanceForTest(); } + @Before + public void resetCMS() + { + ServerTestUtils.resetCMS(); + } + @Test public void testRac() throws IOException, ConfigurationException { @@ -60,18 +64,14 @@ public class AlibabaCloudSnitchTest doReturn(az).when(spiedConnector).apiCall(any()); + // for registering a new node, location is obtained from the cloud metadata service + AlibabaCloudLocationProvider locationProvider = new AlibabaCloudLocationProvider(spiedConnector); + assertEquals("cn-hangzhou", locationProvider.initialLocation().datacenter); + assertEquals("f", locationProvider.initialLocation().rack); + AlibabaCloudSnitch snitch = new AlibabaCloudSnitch(spiedConnector); - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7"); - - Token t1 = ClusterMetadata.current().partitioner.getRandomToken(); - ClusterMetadataTestHelper.addEndpoint(nonlocal, t1, "cn-shanghai", "a"); - - assertEquals("cn-shanghai", snitch.getDatacenter(nonlocal)); - assertEquals("a", snitch.getRack(nonlocal)); - - assertEquals("cn-hangzhou", snitch.getDatacenter(local)); - assertEquals("f", snitch.getRack(local)); + assertEquals("cn-hangzhou", snitch.getLocalDatacenter()); + assertEquals("f", snitch.getLocalRack()); } @Test @@ -83,9 +83,13 @@ public class AlibabaCloudSnitchTest doReturn(az).when(spiedConnector).apiCall(any()); + // for registering a new node, location is obtained from the cloud metadata service + AlibabaCloudLocationProvider locationProvider = new AlibabaCloudLocationProvider(spiedConnector); + assertEquals("us-east", locationProvider.initialLocation().datacenter); + assertEquals("1a", locationProvider.initialLocation().rack); + AlibabaCloudSnitch snitch = new AlibabaCloudSnitch(spiedConnector); - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - assertEquals("us-east", snitch.getDatacenter(local)); - assertEquals("1a", snitch.getRack(local)); + assertEquals("us-east", snitch.getLocalDatacenter()); + assertEquals("1a", snitch.getLocalRack()); } } diff --git a/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java b/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java index 2bec9b8d29..9e9ff605df 100644 --- a/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java +++ b/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java @@ -86,23 +86,6 @@ public class AssureSufficientLiveNodesTest DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); ServerTestUtils.prepareServerNoRegister(); // Register peers with expected DC for NetworkTopologyStrategy. - - // TODO shouldn't require the snitch setup - DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch() - { - public String getRack(InetAddressAndPort endpoint) - { - byte[] address = endpoint.addressBytes; - return "rake" + address[1]; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - byte[] address = endpoint.addressBytes; - return "datacenter" + address[1]; - } - }); - List instances = ImmutableList.of( // datacenter 1 InetAddressAndPort.getByName("127.1.0.255"), InetAddressAndPort.getByName("127.1.0.254"), InetAddressAndPort.getByName("127.1.0.253"), @@ -118,6 +101,10 @@ public class AssureSufficientLiveNodesTest String rack = "rake" + ip.addressBytes[1]; ClusterMetadataTestHelper.addEndpoint(ip, new Murmur3Partitioner.LongToken(i), dc, rack); } + // Need to register the local endpoint as ReplicaLayout::for(Range|Token)ReadLiveSorted sorts replicas by proximity to + // the local node. We use constants from SimpleSnitch to preserve previous test behaviour + InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + ClusterMetadataTestHelper.register(local, SimpleLocationProvider.LOCATION); } @Test @@ -268,13 +255,13 @@ public class AssureSufficientLiveNodesTest for (int i = 0; i < loopCount; i++) { // reset the keyspace -// racedKs.setMetadata(initKsMeta); + SchemaTestUtil.addOrUpdateKeyspace(initKsMeta); CountDownLatch trigger = new CountDownLatch(1); // starts 2 runnables that could race Future f1 = es.submit(() -> { Uninterruptibles.awaitUninterruptibly(trigger); // Update replication strategy -// racedKs.setMetadata(alterToKsMeta); + SchemaTestUtil.addOrUpdateKeyspace(alterToKsMeta); }); Future f2 = es.submit(() -> { Uninterruptibles.awaitUninterruptibly(trigger); diff --git a/test/unit/org/apache/cassandra/locator/AzureConnectorMockingTest.java b/test/unit/org/apache/cassandra/locator/AzureConnectorMockingTest.java index 1fa2eb6887..17aa357979 100644 --- a/test/unit/org/apache/cassandra/locator/AzureConnectorMockingTest.java +++ b/test/unit/org/apache/cassandra/locator/AzureConnectorMockingTest.java @@ -34,9 +34,9 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMoc import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; -import static org.apache.cassandra.locator.AzureSnitch.API_VERSION_PROPERTY_KEY; -import static org.apache.cassandra.locator.AzureSnitch.METADATA_HEADER; -import static org.apache.cassandra.locator.AzureSnitch.METADATA_QUERY_TEMPLATE; +import static org.apache.cassandra.locator.AzureCloudLocationProvider.API_VERSION_PROPERTY_KEY; +import static org.apache.cassandra.locator.AzureCloudLocationProvider.METADATA_HEADER; +import static org.apache.cassandra.locator.AzureCloudLocationProvider.METADATA_QUERY_TEMPLATE; import static org.junit.Assert.assertEquals; public class AzureConnectorMockingTest @@ -60,8 +60,12 @@ public class AzureConnectorMockingTest p.setProperty(METADATA_URL_PROPERTY, "http://127.0.0.1:8080"); SnitchProperties snitchProperties = new SnitchProperties(p); - AzureSnitch azureSnitch = new AzureSnitch(new DefaultCloudMetadataServiceConnector(snitchProperties)); + DefaultCloudMetadataServiceConnector connector = new DefaultCloudMetadataServiceConnector(snitchProperties); + AzureCloudLocationProvider locationProvider = new AzureCloudLocationProvider(connector); + assertEquals("rack-1", locationProvider.initialLocation().rack); + assertEquals("PolandCentral", locationProvider.initialLocation().datacenter); + AzureSnitch azureSnitch = new AzureSnitch(connector); assertEquals("rack-1", azureSnitch.getLocalRack()); assertEquals("PolandCentral", azureSnitch.getLocalDatacenter()); } @@ -83,8 +87,12 @@ public class AzureConnectorMockingTest p.setProperty(API_VERSION_PROPERTY_KEY, "2021-12-14"); SnitchProperties snitchProperties = new SnitchProperties(p); - AzureSnitch azureSnitch = new AzureSnitch(new DefaultCloudMetadataServiceConnector(snitchProperties)); + DefaultCloudMetadataServiceConnector connector = new DefaultCloudMetadataServiceConnector(snitchProperties); + AzureCloudLocationProvider locationProvider = new AzureCloudLocationProvider(connector); + assertEquals("rack-5", locationProvider.initialLocation().rack); + assertEquals("PolandCentral", locationProvider.initialLocation().datacenter); + AzureSnitch azureSnitch = new AzureSnitch(connector); assertEquals("rack-5", azureSnitch.getLocalRack()); assertEquals("PolandCentral", azureSnitch.getLocalDatacenter()); } diff --git a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java index 012fd938ce..338a7f08af 100644 --- a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java @@ -24,16 +24,13 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; -import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.StubClusterMetadataService; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION; import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; @@ -47,15 +44,14 @@ public class CloudstackSnitchTest @BeforeClass public static void setup() throws Exception { - GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true); DatabaseDescriptor.daemonInitialization(); - ClusterMetadataTestHelper.setInstanceForTest(); } @Before public void resetCMS() { - ServerTestUtils.resetCMS(); + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); } @Test @@ -68,18 +64,14 @@ public class CloudstackSnitchTest doReturn(az).when(spiedConnector).apiCall(any()); + // for registering a new node, location is obtained from the cloud metadata service + CloudstackLocationProvider locationProvider = new CloudstackLocationProvider(spiedConnector); + assertEquals("ch-gva", locationProvider.initialLocation().datacenter); + assertEquals("1", locationProvider.initialLocation().rack); + CloudstackSnitch snitch = new CloudstackSnitch(spiedConnector); - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7"); - - Token t1 = ClusterMetadata.current().partitioner.getRandomToken(); - ClusterMetadataTestHelper.addEndpoint(nonlocal, t1, "ch-zrh", "2"); - - assertEquals("ch-zrh", snitch.getDatacenter(nonlocal)); - assertEquals("2", snitch.getRack(nonlocal)); - - assertEquals("ch-gva", snitch.getDatacenter(local)); - assertEquals("1", snitch.getRack(local)); + assertEquals("ch-gva", snitch.getLocalDatacenter()); + assertEquals("1", snitch.getLocalRack()); } @Test @@ -92,11 +84,13 @@ public class CloudstackSnitchTest doReturn(az).when(spiedConnector).apiCall(any()); + // for registering a new node, location is obtained from the cloud metadata service + CloudstackLocationProvider locationProvider = new CloudstackLocationProvider(spiedConnector); + assertEquals("us-east", locationProvider.initialLocation().datacenter); + assertEquals("1a", locationProvider.initialLocation().rack); + CloudstackSnitch snitch = new CloudstackSnitch(spiedConnector); - - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - - assertEquals("us-east", snitch.getDatacenter(local)); - assertEquals("1a", snitch.getRack(local)); + assertEquals("us-east", snitch.getLocalDatacenter()); + assertEquals("1a", snitch.getLocalRack()); } } diff --git a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java index 310be2a6b0..e1cf4ab517 100644 --- a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java @@ -87,8 +87,8 @@ public class DynamicEndpointSnitchTest // do this because SS needs to be initialized before DES can work properly. DatabaseDescriptor.setDynamicBadnessThreshold(0.1); StorageService.instance.unsafeInitialize(); - SimpleSnitch ss = new SimpleSnitch(); - DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode())); + NodeProximity proximity = new NoOpProximity(); + DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(proximity, String.valueOf(proximity.hashCode())); InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); InetAddressAndPort host1 = InetAddressAndPort.getByName("127.0.0.2"); InetAddressAndPort host2 = InetAddressAndPort.getByName("127.0.0.3"); diff --git a/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java b/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java index ef54a60c43..cb8476da04 100644 --- a/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/Ec2SnitchTest.java @@ -18,30 +18,30 @@ package org.apache.cassandra.locator; +import java.io.IOException; import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.Map; -import org.junit.AfterClass; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.StubClusterMetadataService; +import org.apache.cassandra.tcm.membership.Directory; +import org.apache.cassandra.utils.Pair; import org.mockito.stubbing.Answer; -import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION; import static org.apache.cassandra.locator.Ec2MultiRegionSnitch.PRIVATE_IP_QUERY; import static org.apache.cassandra.locator.Ec2MultiRegionSnitch.PUBLIC_IP_QUERY; import static org.apache.cassandra.locator.Ec2Snitch.EC2_NAMING_LEGACY; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -62,22 +62,14 @@ public class Ec2SnitchTest @BeforeClass public static void setup() throws Exception { - GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true); DatabaseDescriptor.daemonInitialization(); - ClusterMetadataTestHelper.setInstanceForTest(); - } - - - @AfterClass - public static void tearDown() - { - StorageService.instance.stopClient(); } @Before public void resetCMS() { - ServerTestUtils.resetCMS(); + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); } @Test @@ -86,6 +78,10 @@ public class Ec2SnitchTest Ec2MetadataServiceConnector connectorMock = mock(Ec2MetadataServiceConnector.class); doReturn("us-east-1d").when(connectorMock).apiCall(anyString()); doReturn(legacySnitchProps).when(connectorMock).getProperties(); + + Ec2LocationProvider locationProvider = new Ec2LocationProvider(connectorMock); + testLegacyRacInternal(locationProvider); + Ec2Snitch snitch = new Ec2Snitch(connectorMock); testLegacyRacInternal(snitch); } @@ -100,6 +96,9 @@ public class Ec2SnitchTest return (PUBLIC_IP_QUERY.equals(query) || PRIVATE_IP_QUERY.equals(query)) ? "127.0.0.1" : "us-east-1d"; }).when(spy).apiCall(anyString()); + Ec2LocationProvider locationProvider = new Ec2LocationProvider(spy); + testLegacyRacInternal(locationProvider); + Ec2Snitch snitch = new Ec2MultiRegionSnitch(spy); testLegacyRacInternal(snitch); } @@ -110,6 +109,7 @@ public class Ec2SnitchTest Ec2MetadataServiceConnector spy = spy(Ec2MetadataServiceConnector.create(legacySnitchProps)); doReturn(legacySnitchProps).when(spy).getProperties(); doReturn("us-east-2d").when(spy).apiCall(anyString()); + testLegacyNewRegionsInternal(new Ec2LocationProvider(spy)); testLegacyNewRegionsInternal(new Ec2Snitch(spy)); } @@ -123,6 +123,7 @@ public class Ec2SnitchTest return (PUBLIC_IP_QUERY.equals(query) || PRIVATE_IP_QUERY.equals(query)) ? "127.0.0.1" : "us-east-2d"; }).when(spy).apiCall(anyString()); + testLegacyNewRegionsInternal(new Ec2LocationProvider(spy)); testLegacyNewRegionsInternal(new Ec2MultiRegionSnitch(spy)); } @@ -132,142 +133,165 @@ public class Ec2SnitchTest Ec2MetadataServiceConnector connectorMock = mock(Ec2MetadataServiceConnector.class); when(connectorMock.apiCall(anyString())).thenReturn("us-east-2d"); when(connectorMock.getProperties()).thenReturn(new SnitchProperties()); + + Ec2LocationProvider locationProvider = new Ec2LocationProvider(connectorMock); + assertEquals("us-east-2", locationProvider.initialLocation().datacenter); + assertEquals("us-east-2d", locationProvider.initialLocation().rack); + Ec2Snitch snitch = new Ec2Snitch(connectorMock); - - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - - assertEquals("us-east-2", snitch.getDatacenter(local)); - assertEquals("us-east-2d", snitch.getRack(local)); + // for registering a new node, location is obtained from the cloud metadata service + assertEquals("us-east-2", snitch.getLocalDatacenter()); + assertEquals("us-east-2d", snitch.getLocalRack()); Ec2MetadataServiceConnector multiRegionConnectorMock = mock(Ec2MetadataServiceConnector.class); - when(connectorMock.getProperties()).thenReturn(new SnitchProperties()); + when(multiRegionConnectorMock.getProperties()).thenReturn(new SnitchProperties()); when(multiRegionConnectorMock.apiCall(anyString())).then((Answer) invocation -> { String query = invocation.getArgument(0); return (PUBLIC_IP_QUERY.equals(query) || PRIVATE_IP_QUERY.equals(query)) ? "127.0.0.1" : "us-east-2d"; }); - assertEquals("us-east-2", snitch.getDatacenter(local)); - assertEquals("us-east-2d", snitch.getRack(local)); + Ec2LocationProvider mrLocationProvider = new Ec2LocationProvider(connectorMock); + assertEquals("us-east-2", mrLocationProvider.initialLocation().datacenter); + assertEquals("us-east-2d", mrLocationProvider.initialLocation().rack); + + Ec2MultiRegionSnitch mrSnitch = new Ec2MultiRegionSnitch(multiRegionConnectorMock); + // for registering a new node, location is obtained from the cloud metadata service + assertEquals("us-east-2", mrSnitch.getLocalDatacenter()); + assertEquals("us-east-2d", mrSnitch.getLocalRack()); } @Test - public void validateDatacenter_RequiresLegacy_CorrectAmazonName() + public void validateDatacenter_RequiresLegacy_CorrectAmazonName() throws IOException { - Set datacenters = new HashSet<>(); - datacenters.add("us-east-1"); - assertTrue(Ec2Snitch.validate(datacenters, Collections.emptySet(), true)); + // legacy scheme / standard dc / ignore racks + assertValid(true, "us-east-1"); } @Test - public void validateDatacenter_RequiresLegacy_LegacyName() + public void validateDatacenter_RequiresLegacy_LegacyName() throws IOException { - Set datacenters = new HashSet<>(); - datacenters.add("us-east"); - assertTrue(Ec2Snitch.validate(datacenters, Collections.emptySet(), true)); + // legacy scheme / legacy dc / ignore racks + assertValid(true, "us-east"); } @Test - public void validate_RequiresLegacy_HappyPath() + public void validate_RequiresLegacy_HappyPath() throws IOException { - Set datacenters = new HashSet<>(); - datacenters.add("us-east"); - Set racks = new HashSet<>(); - racks.add("1a"); - assertTrue(Ec2Snitch.validate(datacenters, racks, true)); + // legacy scheme / legacy dc / legacy rack + assertValid(true, "us-east", "1a"); } @Test - public void validate_RequiresLegacy_HappyPathWithDCSuffix() + public void validate_RequiresLegacy_HappyPathWithDCSuffix() throws IOException { - Set datacenters = new HashSet<>(); - datacenters.add("us-east_CUSTOM_SUFFIX"); - Set racks = new HashSet<>(); - racks.add("1a"); - assertTrue(Ec2Snitch.validate(datacenters, racks, true)); + // legacy scheme / custom suffix dc / legacy rack + assertValid(true, "us-east_CUSTOM_SUFFIX", "1a"); } @Test - public void validateRack_RequiresAmazonName_CorrectAmazonName() + public void validateRack_RequiresAmazonName_CorrectAmazonName() throws IOException { - Set racks = new HashSet<>(); - racks.add("us-east-1a"); - assertTrue(Ec2Snitch.validate(Collections.emptySet(), racks, false)); + // standard scheme / ignore dc / non legacy rack + assertValid(false, "", "us-east-1a"); + } + + @Test(expected = ConfigurationException.class) + public void validateRack_RequiresAmazonName_LegacyName() throws IOException + { + // standard scheme / ignore dc / legacy rack + assertValid(false, "", "1a"); } @Test - public void validateRack_RequiresAmazonName_LegacyName() + public void validate_RequiresAmazonName_HappyPath() throws IOException { - Set racks = new HashSet<>(); - racks.add("1a"); - assertFalse(Ec2Snitch.validate(Collections.emptySet(), racks, false)); + // standard scheme / standard dc / standard rack + assertValid(false, "us-east-1", "us-east-1a"); } @Test - public void validate_RequiresAmazonName_HappyPath() + public void validate_RequiresAmazonName_HappyPathWithDCSuffix() throws IOException { - Set datacenters = new HashSet<>(); - datacenters.add("us-east-1"); - Set racks = new HashSet<>(); - racks.add("us-east-1a"); - assertTrue(Ec2Snitch.validate(datacenters, racks, false)); - } - - @Test - public void validate_RequiresAmazonName_HappyPathWithDCSuffix() - { - Set datacenters = new HashSet<>(); - datacenters.add("us-east-1_CUSTOM_SUFFIX"); - Set racks = new HashSet<>(); - racks.add("us-east-1a"); - assertTrue(Ec2Snitch.validate(datacenters, racks, false)); + // standard scheme / custom suffix dc / standard rack + assertValid(false, "us-east-1_CUSTOM_SUFFIX", "us-east-1a"); } /** * Validate upgrades in legacy mode for regions that didn't change name between the standard and legacy modes. */ @Test - public void validate_RequiresLegacy_DCValidStandardAndLegacy() + public void validate_RequiresLegacy_DCValidStandardAndLegacy() throws IOException { - Set datacenters = new HashSet<>(); - datacenters.add("us-west-2"); - Set racks = new HashSet<>(); - racks.add("2a"); - racks.add("2b"); - assertTrue(Ec2Snitch.validate(datacenters, racks, true)); + // legacy scheme / standard dc / legacy racks + assertValid(true, "us-west-2", "2a", "2b"); } /** * Check that racks names are enough to detect a mismatch in naming conventions. */ - @Test - public void validate_RequiresLegacy_RackInvalidForLegacy() + @Test(expected = ConfigurationException.class) + public void validate_RequiresLegacy_RackInvalidForLegacy() throws Exception { - Set datacenters = new HashSet<>(); - datacenters.add("us-west-2"); - Set racks = new HashSet<>(); - racks.add("us-west-2a"); - assertFalse(Ec2Snitch.validate(datacenters, racks, true)); + // legacy scheme / standard dc / standard rack + assertValid(true, "us-west-2", "us-west-2a"); + } + + private static void assertValid(boolean useLegacy, String datacenter, String...racks) throws IOException + { + SnitchProperties properties = new SnitchProperties(Pair.create(Ec2LocationProvider.SNITCH_PROP_NAMING_SCHEME, + useLegacy ? Ec2LocationProvider.EC2_NAMING_LEGACY + : Ec2LocationProvider.EC2_NAMING_STANDARD)); + Ec2MetadataServiceConnector connectorMock = mock(Ec2MetadataServiceConnector.class); + when(connectorMock.getProperties()).thenReturn(properties); + Ec2LocationProvider provider = new Ec2LocationProvider(connectorMock); + + Multimap dcRacks = HashMultimap.create(); + for (String rack : racks) + dcRacks.put(rack, null); // endpoints are used + Directory directory = new Directory() + { + @Override + public Map> allDatacenterRacks() + { + return Collections.singletonMap(datacenter, dcRacks); + } + + @Override + public Multimap datacenterRacks(String datacenter) + { + return dcRacks; + } + }; + ClusterMetadata metadata = new ClusterMetadata(Murmur3Partitioner.instance, directory); + + provider.validate(metadata); + } + + private void testLegacyRacInternal(Ec2LocationProvider provider) throws Exception + { + // for registering a new node, location is obtained from the cloud metadata service + assertEquals("us-east", provider.initialLocation().datacenter); + assertEquals("1d", provider.initialLocation().rack); } private void testLegacyRacInternal(Ec2Snitch snitch) throws Exception { - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7"); + // for registering a new node, location is obtained from the cloud metadata service + assertEquals("us-east", snitch.getLocalDatacenter()); + assertEquals("1d", snitch.getLocalRack()); + } - Token t1 = ClusterMetadata.current().partitioner.getRandomToken(); - ClusterMetadataTestHelper.addEndpoint(nonlocal, t1, "us-west", "1a"); - - assertEquals("us-west", snitch.getDatacenter(nonlocal)); - assertEquals("1a", snitch.getRack(nonlocal)); - - assertEquals("us-east", snitch.getDatacenter(local)); - assertEquals("1d", snitch.getRack(local)); + private void testLegacyNewRegionsInternal(Ec2LocationProvider provider) throws Exception + { + // for registering a new node, location is obtained from the cloud metadata service + assertEquals("us-east-2", provider.initialLocation().datacenter); + assertEquals("2d", provider.initialLocation().rack); } private void testLegacyNewRegionsInternal(Ec2Snitch snitch) throws Exception { - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - assertEquals("us-east-2", snitch.getDatacenter(local)); - assertEquals("2d", snitch.getRack(local)); + // for registering a new node, location is obtained from the cloud metadata service + assertEquals("us-east-2", snitch.getLocalDatacenter()); + assertEquals("2d", snitch.getLocalRack()); } } diff --git a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java index b447034e95..277990a3cb 100644 --- a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java @@ -21,20 +21,19 @@ package org.apache.cassandra.locator; import java.io.IOException; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.DefaultCloudMetadataServiceConnector; -import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.StubClusterMetadataService; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DISABLE_THREAD_VALIDATION; import static org.apache.cassandra.locator.AbstractCloudMetadataServiceConnector.METADATA_URL_PROPERTY; -import static org.apache.cassandra.locator.AlibabaCloudSnitch.DEFAULT_METADATA_SERVICE_URL; +import static org.apache.cassandra.locator.AlibabaCloudLocationProvider.DEFAULT_METADATA_SERVICE_URL; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; @@ -46,9 +45,14 @@ public class GoogleCloudSnitchTest @BeforeClass public static void setup() throws Exception { - GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true); DatabaseDescriptor.daemonInitialization(); - ClusterMetadataTestHelper.setInstanceForTest(); + } + + @Before + public void resetCMS() + { + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); } @Test @@ -61,18 +65,14 @@ public class GoogleCloudSnitchTest doReturn(az).when(spiedConnector).apiCall(any(), anyMap()); + // for registering a new node, location is obtained from the cloud metadata service + GoogleCloudLocationProvider locationProvider = new GoogleCloudLocationProvider(spiedConnector); + assertEquals("us-central1", locationProvider.initialLocation().datacenter); + assertEquals("a", locationProvider.initialLocation().rack); + GoogleCloudSnitch snitch = new GoogleCloudSnitch(spiedConnector); - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - InetAddressAndPort nonlocal = InetAddressAndPort.getByName("127.0.0.7"); - - Token t1 = ClusterMetadata.current().partitioner.getRandomToken(); - ClusterMetadataTestHelper.addEndpoint(nonlocal, t1, "europe-west1", "a"); - - assertEquals("europe-west1", snitch.getDatacenter(nonlocal)); - assertEquals("a", snitch.getRack(nonlocal)); - - assertEquals("us-central1", snitch.getDatacenter(local)); - assertEquals("a", snitch.getRack(local)); + assertEquals("us-central1", snitch.getLocalDatacenter()); + assertEquals("a", snitch.getLocalRack()); } @Test @@ -85,9 +85,13 @@ public class GoogleCloudSnitchTest doReturn(az).when(spiedConnector).apiCall(any(), anyMap()); + // local endpoint is not yet registered, so location is obtained from the cloud metadata service connector + GoogleCloudLocationProvider locationProvider = new GoogleCloudLocationProvider(spiedConnector); + assertEquals("asia-east1", locationProvider.initialLocation().datacenter); + assertEquals("a", locationProvider.initialLocation().rack); + GoogleCloudSnitch snitch = new GoogleCloudSnitch(spiedConnector); - InetAddressAndPort local = InetAddressAndPort.getByName("127.0.0.1"); - assertEquals("asia-east1", snitch.getDatacenter(local)); - assertEquals("a", snitch.getRack(local)); + assertEquals("asia-east1", snitch.getLocalDatacenter()); + assertEquals("a", snitch.getLocalRack()); } } diff --git a/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java index dcd182d23e..c8ff6ea45a 100644 --- a/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java @@ -18,13 +18,14 @@ package org.apache.cassandra.locator; -import java.net.UnknownHostException; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.StubClusterMetadataService; import static org.junit.Assert.*; @@ -33,34 +34,25 @@ import static org.junit.Assert.*; */ public class GossipingPropertyFileSnitchTest { - @BeforeClass - public static void setupDD() + public static void setup() throws Exception { DatabaseDescriptor.daemonInitialization(); } - public static void checkEndpoint(final AbstractNetworkTopologySnitch snitch, - final String endpointString, final String expectedDatacenter, - final String expectedRack) + @Before + public void resetCMS() { - final InetAddressAndPort endpoint; - try - { - endpoint = InetAddressAndPort.getByName(endpointString); - } - catch (UnknownHostException e) - { - throw new RuntimeException(e); - } - assertEquals(expectedDatacenter, snitch.getDatacenter(endpoint)); - assertEquals(expectedRack, snitch.getRack(endpoint)); + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); } @Test - public void testLoadConfig() throws Exception + public void testLoadConfig() { final GossipingPropertyFileSnitch snitch = new GossipingPropertyFileSnitch(); - checkEndpoint(snitch, FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort(), "DC1", "RAC1"); + // for registering a new node, location is obtained from the snitch config + assertEquals("DC1", snitch.getLocalDatacenter()); + assertEquals("RAC1", snitch.getLocalRack()); } } diff --git a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java index 9a97999827..fc235526fc 100644 --- a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java @@ -45,14 +45,18 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.tcm.compatibility.TokenRingUtils; +import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.VersionedEndpoints; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.locator.NetworkTopologyStrategy.REPLICATION_FACTOR; import static org.apache.cassandra.locator.Replica.fullReplica; import static org.apache.cassandra.locator.Replica.transientReplica; +import static org.apache.cassandra.locator.SimpleLocationProvider.LOCATION; import static org.junit.Assert.assertTrue; public class NetworkTopologyStrategyTest @@ -78,8 +82,6 @@ public class NetworkTopologyStrategyTest @Test public void testProperties() throws IOException, ConfigurationException { - IEndpointSnitch snitch = new PropertyFileSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); createDummyTokens(true); ClusterMetadataTestHelper.createKeyspace("CREATE KEYSPACE " + KEYSPACE + " WITH REPLICATION = {" + @@ -104,8 +106,6 @@ public class NetworkTopologyStrategyTest @Test public void testPropertiesWithEmptyDC() throws IOException, ConfigurationException { - IEndpointSnitch snitch = new PropertyFileSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); createDummyTokens(false); Map configOptions = new HashMap<>(); @@ -132,9 +132,6 @@ public class NetworkTopologyStrategyTest int[] dcEndpoints = new int[]{128, 256, 512}; int[] dcReplication = new int[]{2, 6, 6}; - IEndpointSnitch snitch = new RackInferringSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); - Map configOptions = new HashMap(); Multimap tokens = HashMultimap.create(); @@ -152,7 +149,8 @@ public class NetworkTopologyStrategyTest StringToken token = new StringToken(String.format("%02x%02x%02x", ep, rack, dc)); logger.debug("adding node {} at {}", address, token); tokens.put(address, token); - ClusterMetadataTestHelper.addEndpoint(address, token, snitch.getDatacenter(address), snitch.getRack(address)); + Location location = RackInferringSnitch.inferLocation(address); + ClusterMetadataTestHelper.addEndpoint(address, token, location.datacenter, location.rack); } } } @@ -218,8 +216,7 @@ public class NetworkTopologyStrategyTest { ServerTestUtils.resetCMS(); Random rand = new Random(run); - IEndpointSnitch snitch = generateSnitch(datacenters, nodes, rand); - DatabaseDescriptor.setEndpointSnitch(snitch); + Locator locator = generateLocator(datacenters, nodes, rand); for (int i = 0; i < NODES; ++i) // Nodes { @@ -228,14 +225,27 @@ public class NetworkTopologyStrategyTest { tokens.add(Murmur3Partitioner.instance.getRandomToken(rand)); } - ClusterMetadataTestHelper.addEndpoint(nodes.get(i), tokens, snitch.getDatacenter(nodes.get(i)), snitch.getRack(nodes.get(i))); + // Here we fake the registration status because we want all the nodes to be registered in cluster + // metadata using the locations we setup in generateLocator. This registration occurs as a part of + // the addEndpoint call here and behaves as expected for all nodes _except_ the one with the address + // which matches the local broadcast address (i.e. 127.0.0.1, which is #2 in the list of nodes). + // The location we want this to be registered with is {DC: rf5_1, rack: 3}, but while + // RegistrationStatus.instance indicates that the node is yet to be registered, the Locator will + // correctly return the initialization location obtained from + // DatabaseDescriptor::getInitialLocationProvider, which ultimately resolves to + // SimpleLocationProvider (because test/conf/cassandra.yaml specifies use of SimpleSnitch) and so + // we register that one node with the location {DC: datacenter1, rack: rack1}. + // This is purely an artefact of the contrived testing setup and in more realistic scenarios, + // including the majority of tests, isn't an issue. + RegistrationStatus.instance.onRegistration(); + ClusterMetadataTestHelper.addEndpoint(nodes.get(i), tokens, locator.location(nodes.get(i))); } - testEquivalence(ClusterMetadata.current(), snitch, datacenters, rand); + testEquivalence(ClusterMetadata.current(), locator, datacenters, rand); } } } - void testEquivalence(ClusterMetadata metadata, IEndpointSnitch snitch, Map datacenters, Random rand) + void testEquivalence(ClusterMetadata metadata, Locator locator, Map datacenters, Random rand) { NetworkTopologyStrategy nts = new NetworkTopologyStrategy("ks", datacenters.entrySet() @@ -244,7 +254,7 @@ public class NetworkTopologyStrategyTest for (int i=0; i<1000; ++i) { Token token = Murmur3Partitioner.instance.getRandomToken(rand); - List expected = calculateNaturalEndpoints(token, metadata, datacenters, snitch); + List expected = calculateNaturalEndpoints(token, metadata, datacenters, locator); List actual = new ArrayList<>(nts.calculateNaturalReplicas(token, metadata).endpoints()); if (endpointsDiffer(expected, actual)) { @@ -270,10 +280,11 @@ public class NetworkTopologyStrategyTest return !s1.equals(s2); } - IEndpointSnitch generateSnitch(Map datacenters, Collection nodes, Random rand) + Locator generateLocator(Map datacenters, Collection nodes, Random rand) { - final Map nodeToRack = new HashMap<>(); - final Map nodeToDC = new HashMap<>(); + final Map nodeToRack = new HashMap<>(); + final Map nodeToDC = new HashMap<>(); + final Map epToId = new HashMap<>(); Map> racksPerDC = new HashMap<>(); datacenters.forEach((dc, rf) -> racksPerDC.put(dc, randomRacks(rf, rand))); int rf = datacenters.values().stream().mapToInt(x -> x).sum(); @@ -285,27 +296,33 @@ public class NetworkTopologyStrategyTest dcs[pos++] = dce.getKey(); } + int id = 0; for (InetAddressAndPort node : nodes) { String dc = dcs[rand.nextInt(rf)]; List racks = racksPerDC.get(dc); String rack = racks.get(rand.nextInt(racks.size())); - nodeToRack.put(node, rack); - nodeToDC.put(node, dc); + NodeId nodeId = new NodeId(++id); + nodeToRack.put(nodeId, rack); + nodeToDC.put(nodeId, dc); + epToId.put(node, nodeId); } - return new AbstractNetworkTopologySnitch() + Directory dir = new Directory() { - public String getRack(InetAddressAndPort endpoint) + @Override + public NodeId peerId(InetAddressAndPort endpoint) { - return nodeToRack.get(endpoint); + return epToId.get(endpoint); } - public String getDatacenter(InetAddressAndPort endpoint) + @Override + public Location location(NodeId id) { - return nodeToDC.get(endpoint); + return new Location(nodeToDC.get(id), nodeToRack.get(id)); } }; + return Locator.usingDirectory(dir); } private List randomRacks(int rf, Random rand) @@ -318,7 +335,7 @@ public class NetworkTopologyStrategyTest } // Copy of older endpoints calculation algorithm for comparison - public static List calculateNaturalEndpoints(Token searchToken, ClusterMetadata metadata, Map datacenters, IEndpointSnitch snitch) + public static List calculateNaturalEndpoints(Token searchToken, ClusterMetadata metadata, Map datacenters, Locator locator) { // we want to preserve insertion order so that the first added endpoint becomes primary Set replicas = new LinkedHashSet<>(); @@ -349,7 +366,8 @@ public class NetworkTopologyStrategyTest { Token next = tokenIter.next(); InetAddressAndPort ep = metadata.directory.endpoint(metadata.tokenMap.owner(next)); - String dc = snitch.getDatacenter(ep); + Location location = locator.location(ep); + String dc = location.datacenter; // have we already found all replicas for this dc? if (!datacenters.containsKey(dc) || hasSufficientReplicas(dc, dcReplicas, allEndpoints, datacenters)) continue; @@ -361,7 +379,7 @@ public class NetworkTopologyStrategyTest } else { - String rack = snitch.getRack(ep); + String rack = location.rack; // is this a new rack? if (seenRacks.get(dc).contains(rack)) { @@ -424,21 +442,18 @@ public class NetworkTopologyStrategyTest { try (WithPartitioner m3p = new WithPartitioner(Murmur3Partitioner.instance)) { - IEndpointSnitch snitch = new SimpleSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); - List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), InetAddressAndPort.getByName("127.0.0.2"), InetAddressAndPort.getByName("127.0.0.3"), InetAddressAndPort.getByName("127.0.0.4")); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(0), tk(100), SimpleSnitch.DATA_CENTER_NAME, SimpleSnitch.RACK_NAME); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(1), tk(200), SimpleSnitch.DATA_CENTER_NAME, SimpleSnitch.RACK_NAME); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(2), tk(300), SimpleSnitch.DATA_CENTER_NAME, SimpleSnitch.RACK_NAME); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(3), tk(400), SimpleSnitch.DATA_CENTER_NAME, SimpleSnitch.RACK_NAME); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(0), tk(100), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(1), tk(200), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(2), tk(300), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(3), tk(400), LOCATION); Map configOptions = new HashMap<>(); - configOptions.put(SimpleSnitch.DATA_CENTER_NAME, "3/1"); + configOptions.put(LOCATION.datacenter, "3/1"); NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(KEYSPACE, configOptions); Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(0), range(400, 100)), diff --git a/test/unit/org/apache/cassandra/locator/PendingRangesTest.java b/test/unit/org/apache/cassandra/locator/PendingRangesTest.java index 15b82a686a..45e00a236a 100644 --- a/test/unit/org/apache/cassandra/locator/PendingRangesTest.java +++ b/test/unit/org/apache/cassandra/locator/PendingRangesTest.java @@ -80,7 +80,7 @@ public class PendingRangesTest { ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(ClusterMetadataTestHelper.syncInstanceForTest()); - ClusterMetadataService.instance().log().bootstrap(FBUtilities.getBroadcastAddressAndPort()); + ClusterMetadataService.instance().log().unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); } @Test @@ -598,21 +598,21 @@ public class PendingRangesTest } } - private static IEndpointSnitch snitch() - { - return new AbstractNetworkTopologySnitch() - { - public String getRack(InetAddressAndPort endpoint) - { - return RACK1; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - return DC1; - } - }; - } +// private static IEndpointSnitch snitch() +// { +// return new AbstractNetworkTopologySnitch() +// { +// public String getRack(InetAddressAndPort endpoint) +// { +// return RACK1; +// } +// +// public String getDatacenter(InetAddressAndPort endpoint) +// { +// return DC1; +// } +// }; +// } private static AbstractReplicationStrategy simpleStrategy(int replicationFactor) { diff --git a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java index e5f21e6047..35bdb98d8d 100644 --- a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java @@ -26,24 +26,21 @@ import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.regex.Matcher; import java.util.stream.Collectors; -import com.google.common.collect.ImmutableMap; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.StubClusterMetadataService; -import org.apache.cassandra.tcm.membership.MembershipUtils; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.locator.TopologyFileLocationProvider.PROPERTIES_FILENAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -68,7 +65,7 @@ public class PropertyFileSnitchTest { ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); - String confFile = FBUtilities.resourceToFile(PropertyFileSnitch.SNITCH_PROPERTIES_FILENAME); + String confFile = FBUtilities.resourceToFile(PROPERTIES_FILENAME); effectiveFile = Paths.get(confFile); backupFile = Paths.get(confFile + ".bak"); localAddress = FBUtilities.getBroadcastAddressAndPort(); @@ -77,21 +74,24 @@ public class PropertyFileSnitchTest } @Test - public void localLocationPresentInConfig() throws IOException + public void localLocationsPresentInConfig() throws IOException { replaceConfigFile(Collections.singletonMap(localAddress.getHostAddressAndPort(), "DC1:RAC2")); PropertyFileSnitch snitch = new PropertyFileSnitch(); - assertEquals("DC1", snitch.getDatacenter(localAddress)); - assertEquals("RAC2", snitch.getRack(localAddress)); + // for registering a new node, location is obtained from the snitch config + assertEquals("DC1", snitch.getLocalDatacenter()); + assertEquals("RAC2", snitch.getLocalRack()); } @Test - public void localLocationNotPresentInConfig() throws IOException + public void locationsNotPresentInConfig() throws IOException { - replaceConfigFile(Collections.singletonMap("default", "DEFAULT_DC:DEFAULT_RACK")); + replaceConfigFile(Collections.singletonMap("default", "XXX_DEFAULT_DC:XXX_DEFAULT_RACK")); PropertyFileSnitch snitch = new PropertyFileSnitch(); - assertEquals("DEFAULT_DC", snitch.getDatacenter(localAddress)); - assertEquals("DEFAULT_RACK", snitch.getRack(localAddress)); + + // for registering a new node, location is obtained from the snitch config + assertEquals("XXX_DEFAULT_DC", snitch.getLocalDatacenter()); + assertEquals("XXX_DEFAULT_RACK", snitch.getLocalRack()); } @Test @@ -107,34 +107,11 @@ public class PropertyFileSnitchTest { String expectedMessage = String.format("Snitch definitions at %s do not define a location for this node's " + "broadcast address %s, nor does it provides a default", - PropertyFileSnitch.SNITCH_PROPERTIES_FILENAME, localAddress); + PROPERTIES_FILENAME, localAddress); assertTrue(e.getMessage().contains(expectedMessage)); } } - @Test - public void configContainsRemoteConfig() throws IOException - { - // Locations of remote peers should not be accessible from this snitch unless - // they are present in ClusterMetadata - Random r = new Random(System.nanoTime()); - InetAddressAndPort peer = MembershipUtils.endpoint(99); - replaceConfigFile(ImmutableMap.of(localAddress.getHostAddressAndPort(), "DC1:RAC1", - peer.getHostAddressAndPort(), "OTHER_DC1:OTHER_RAC1")); - PropertyFileSnitch snitch = new PropertyFileSnitch(); - assertEquals("DC1", snitch.getDatacenter(localAddress)); - assertEquals("RAC1", snitch.getRack(localAddress)); - - assertEquals(PropertyFileSnitch.DEFAULT_DC, snitch.getDatacenter(peer)); - assertEquals(PropertyFileSnitch.DEFAULT_RACK, snitch.getRack(peer)); - - // Register peer, causing ClusterMetadata to be updated. Note that the location - // here is not the one in the config file, that should still be irrelevant - ClusterMetadataTestHelper.register(peer, "OTHER_DC2", "OTHER_RAC2"); - assertEquals("OTHER_DC2", snitch.getDatacenter(peer)); - assertEquals("OTHER_RAC2", snitch.getRack(peer)); - } - @After public void restoreOrigConfigFile() throws IOException { diff --git a/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java b/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java index d5a0ec36a2..aa629eba22 100644 --- a/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java +++ b/test/unit/org/apache/cassandra/locator/ReconnectableSnitchHelperTest.java @@ -50,7 +50,7 @@ public class ReconnectableSnitchHelperTest DatabaseDescriptor.setInternodeAuthenticator(MessagingServiceTest.ALLOW_NOTHING_AUTHENTICATOR); InetAddressAndPort address = InetAddressAndPort.getByName("127.0.0.250"); //Should tolerate null returns by MS for the connection - ReconnectableSnitchHelper.reconnect(address, address, null, null); + ReconnectableSnitchHelper.reconnect(address, address, null); } @After diff --git a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java index 7f946abbae..c8dbc806dc 100644 --- a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java +++ b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java @@ -18,55 +18,51 @@ package org.apache.cassandra.locator; +import java.util.Map; +import java.util.Set; + import com.google.common.base.Predicates; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.Test; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.StubClusterMetadataService; -import org.junit.Before; -import org.junit.Test; - -import java.util.Map; -import java.util.Set; - import static org.apache.cassandra.locator.Replica.fullReplica; -import static org.apache.cassandra.locator.ReplicaUtils.*; +import static org.apache.cassandra.locator.ReplicaUtils.EP1; +import static org.apache.cassandra.locator.ReplicaUtils.EP2; +import static org.apache.cassandra.locator.ReplicaUtils.EP3; +import static org.apache.cassandra.locator.ReplicaUtils.EP4; +import static org.apache.cassandra.locator.ReplicaUtils.EP5; +import static org.apache.cassandra.locator.ReplicaUtils.EP6; +import static org.apache.cassandra.locator.ReplicaUtils.R1; +import static org.apache.cassandra.locator.ReplicaUtils.assertEquals; +import static org.apache.cassandra.locator.ReplicaUtils.tk; +import static org.apache.cassandra.locator.ReplicaUtils.trans; public class ReplicaPlansTest { - + static Keyspace ks; static { DatabaseDescriptor.daemonInitialization(); } - // TODO replace use of snitch in determining counts per DC with directory lookup - static class Snitch extends AbstractNetworkTopologySnitch + private static Keyspace ks(Set dc1, Map replication) { - final Set dc1; - Snitch(Set dc1) - { - this.dc1 = dc1; - } - @Override - public String getRack(InetAddressAndPort endpoint) - { - return dc1.contains(endpoint) ? "R1" : "R2"; - } - - @Override - public String getDatacenter(InetAddressAndPort endpoint) - { - return dc1.contains(endpoint) ? "DC1" : "DC2"; - } + replication = ImmutableMap.builder().putAll(replication).put("class", "NetworkTopologyStrategy").build(); + Keyspace keyspace = Keyspace.mockKS(KeyspaceMetadata.create("blah", KeyspaceParams.create(false, replication))); + return keyspace; } @Before @@ -74,59 +70,42 @@ public class ReplicaPlansTest { ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); - } - - private static Keyspace ks(Set dc1, Map replication) - { - replication = ImmutableMap.builder().putAll(replication).put("class", "NetworkTopologyStrategy").build(); - Keyspace keyspace = Keyspace.mockKS(KeyspaceMetadata.create("blah", KeyspaceParams.create(false, replication))); - Snitch snitch = new Snitch(dc1); - DatabaseDescriptor.setEndpointSnitch(snitch); - return keyspace; + ClusterMetadataTestHelper.register(EP1, "DC1", "R1"); + ClusterMetadataTestHelper.register(EP2, "DC1", "R1"); + ClusterMetadataTestHelper.register(EP3, "DC1", "R1"); + ClusterMetadataTestHelper.register(EP4, "DC2", "R2"); + ClusterMetadataTestHelper.register(EP5, "DC2", "R2"); + ClusterMetadataTestHelper.register(EP6, "DC2", "R2"); } private static Replica full(InetAddressAndPort ep) { return fullReplica(ep, R1); } - - @Test public void testWriteEachQuorum() { - IEndpointSnitch stash = DatabaseDescriptor.getEndpointSnitch(); final Token token = tk(1L); - try { - { - // all full natural - Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3")); - EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP3), full(EP4), full(EP5), full(EP6)); - EndpointsForToken pending = EndpointsForToken.empty(token); - ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, (cm) -> natural, (cm) -> pending, null, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); - assertEquals(natural, plan.liveAndDown); - assertEquals(natural, plan.live); - assertEquals(natural, plan.contacts()); - } - { - // all natural and up, one transient in each DC - Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3")); - EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6)); - EndpointsForToken pending = EndpointsForToken.empty(token); - ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, (cm) -> natural, (cm) -> pending, Epoch.FIRST, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); - assertEquals(natural, plan.liveAndDown); - assertEquals(natural, plan.live); - EndpointsForToken expectContacts = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP4), full(EP5)); - assertEquals(expectContacts, plan.contacts()); - } + // all full natural + Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3")); + EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP3), full(EP4), full(EP5), full(EP6)); + EndpointsForToken pending = EndpointsForToken.empty(token); + ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, (cm) -> natural, (cm) -> pending, null, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); + assertEquals(natural, plan.liveAndDown); + assertEquals(natural, plan.live); + assertEquals(natural, plan.contacts()); } - finally { - DatabaseDescriptor.setEndpointSnitch(stash); - } - - { - // test simple - + // all natural and up, one transient in each DC + // Note: this is confusing because it looks misconfigured as the Keyspace has never been setup with any + // transient replicas in its replication params. + Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3")); + EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6)); + EndpointsForToken pending = EndpointsForToken.empty(token); + ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, (cm) -> natural, (cm) -> pending, Epoch.FIRST, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); + assertEquals(natural, plan.liveAndDown); + assertEquals(natural, plan.live); + EndpointsForToken expectContacts = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP4), full(EP5)); + assertEquals(expectContacts, plan.contacts()); } } - } diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java index 8357035000..874e50d17a 100644 --- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java @@ -85,12 +85,6 @@ public class SimpleStrategyTest DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); } - @Before - public void resetSnitch() - { - DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch()); - } - @Test public void tryValidKeyspace() { @@ -128,8 +122,6 @@ public class SimpleStrategyTest public void testMultiDCSimpleStrategyEndpoints() throws UnknownHostException { withPartitioner(Murmur3Partitioner.instance); - IEndpointSnitch snitch = new PropertyFileSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); // Topology taken directly from the topology_test.test_size_estimates_multidc dtest that regressed Multimap dc1 = HashMultimap.create(); @@ -275,8 +267,6 @@ public class SimpleStrategyTest public void transientReplica() throws Exception { withPartitioner(Murmur3Partitioner.instance); - IEndpointSnitch snitch = new SimpleSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), InetAddressAndPort.getByName("127.0.0.2"), @@ -321,9 +311,6 @@ public class SimpleStrategyTest expectedEx.expect(ConfigurationException.class); expectedEx.expectMessage("SimpleStrategy requires a replication_factor strategy option."); - IEndpointSnitch snitch = new SimpleSnitch(); - DatabaseDescriptor.setEndpointSnitch(snitch); - List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), InetAddressAndPort.getByName("127.0.0.2"), InetAddressAndPort.getByName("127.0.0.3")); diff --git a/test/unit/org/apache/cassandra/locator/SnitchUtilsTest.java b/test/unit/org/apache/cassandra/locator/SnitchUtilsTest.java index b5b1400988..c203c2b851 100644 --- a/test/unit/org/apache/cassandra/locator/SnitchUtilsTest.java +++ b/test/unit/org/apache/cassandra/locator/SnitchUtilsTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.locator; import org.junit.Test; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.Pair; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -33,13 +34,23 @@ public class SnitchUtilsTest Pair result = SnitchUtils.parseDcAndRack("my-dc-rack1", ""); assertEquals("my-dc", result.left); assertEquals("rack1", result.right); + Location location = SnitchUtils.parseLocation("my-dc-rack1", ""); + assertEquals("my-dc", location.datacenter); + assertEquals("rack1", location.rack); result = SnitchUtils.parseDcAndRack("my-rack", ""); assertEquals("my", result.left); assertEquals("rack", result.right); + location = SnitchUtils.parseLocation("my-rack", ""); + assertEquals("my", location.datacenter); + assertEquals("rack", location.rack); assertThatExceptionOfType(IllegalStateException.class) .isThrownBy(() -> SnitchUtils.parseDcAndRack("myresponse", "")) .withMessage("myresponse does not contain at least one '-' to differentiate between datacenter and rack"); + + assertThatExceptionOfType(IllegalStateException.class) + .isThrownBy(() -> SnitchUtils.parseLocation("myresponse", "")) + .withMessage("myresponse does not contain at least one '-' to differentiate between datacenter and rack"); } } diff --git a/test/unit/org/apache/cassandra/locator/WithPartitioner.java b/test/unit/org/apache/cassandra/locator/WithPartitioner.java index 94a07a6e5f..48738e27fd 100644 --- a/test/unit/org/apache/cassandra/locator/WithPartitioner.java +++ b/test/unit/org/apache/cassandra/locator/WithPartitioner.java @@ -41,6 +41,7 @@ public class WithPartitioner implements AutoCloseable { ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(toRestore); + DatabaseDescriptor.setPartitionerUnsafe(toRestore.metadata().partitioner); } } diff --git a/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java b/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java index 4775fd1d67..0fd923f533 100644 --- a/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java +++ b/test/unit/org/apache/cassandra/net/AsyncStreamingOutputPlusTest.java @@ -24,12 +24,12 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.util.Random; -import org.apache.cassandra.io.util.File; import org.junit.Test; import io.netty.buffer.ByteBuf; import io.netty.channel.embedded.EmbeddedChannel; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.utils.FBUtilities; diff --git a/test/unit/org/apache/cassandra/net/ConnectionTest.java b/test/unit/org/apache/cassandra/net/ConnectionTest.java index 6225ba6454..42c137cc5f 100644 --- a/test/unit/org/apache/cassandra/net/ConnectionTest.java +++ b/test/unit/org/apache/cassandra/net/ConnectionTest.java @@ -122,6 +122,7 @@ public class ConnectionTest { DatabaseDescriptor.daemonInitialization(); ClusterMetadataTestHelper.setInstanceForTest(); + ClusterMetadataTestHelper.register(FBUtilities.getBroadcastAddressAndPort()); CommitLog.instance.start(); } diff --git a/test/unit/org/apache/cassandra/net/HandshakeTest.java b/test/unit/org/apache/cassandra/net/HandshakeTest.java index b66ade056e..c84643497d 100644 --- a/test/unit/org/apache/cassandra/net/HandshakeTest.java +++ b/test/unit/org/apache/cassandra/net/HandshakeTest.java @@ -75,6 +75,8 @@ public class HandshakeTest { DatabaseDescriptor.daemonInitialization(); ClusterMetadataTestHelper.setInstanceForTest(); + ClusterMetadataTestHelper.register(TO_ADDR); + ClusterMetadataTestHelper.register(FROM_ADDR); CommitLog.instance.start(); } diff --git a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java index f1087efb10..95c72f4bc0 100644 --- a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java +++ b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java @@ -117,6 +117,7 @@ public class MessagingServiceTest ClusterMetadataTestHelper.setInstanceForTest(); CommitLog.instance.start(); DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.0.0.1")); + ClusterMetadataTestHelper.register(FBUtilities.getBroadcastAddressAndPort(), "datacenter1", "rack1"); originalAuthenticator = DatabaseDescriptor.getInternodeAuthenticator(); originalServerEncryptionOptions = DatabaseDescriptor.getInternodeMessagingEncyptionOptions(); originalListenAddress = InetAddressAndPort.getByAddressOverrideDefaults(DatabaseDescriptor.getListenAddress(), DatabaseDescriptor.getStoragePort()); diff --git a/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java b/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java index 89549b4f3f..5dae8af48f 100644 --- a/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java +++ b/test/unit/org/apache/cassandra/net/OutboundConnectionSettingsTest.java @@ -18,27 +18,29 @@ package org.apache.cassandra.net; -import java.util.HashMap; -import java.util.Map; import java.util.function.Function; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.commitlog.CommitLog; -import org.apache.cassandra.locator.AbstractEndpointSnitch; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.StubClusterMetadataService; -import static org.apache.cassandra.config.DatabaseDescriptor.getEndpointSnitch; import static org.apache.cassandra.net.OutboundConnectionsTest.LOCAL_ADDR; import static org.apache.cassandra.net.OutboundConnectionsTest.REMOTE_ADDR; public class OutboundConnectionSettingsTest { + private static final String DC1 = "dc1"; + private static final String DC2 = "dc2"; + private static final String RACK = "rack1"; + @BeforeClass public static void before() { @@ -46,6 +48,13 @@ public class OutboundConnectionSettingsTest CommitLog.instance.start(); } + @Before + public void reset() + { + ClusterMetadataService.unsetInstance(); + ClusterMetadataService.setInstance(StubClusterMetadataService.forTesting()); + } + @Test (expected = IllegalArgumentException.class) public void build_SmallSendSize() { @@ -81,65 +90,47 @@ public class OutboundConnectionSettingsTest f.apply(new OutboundConnectionSettings(LOCAL_ADDR)).withDefaults(ConnectionCategory.MESSAGING); } - private static class TestSnitch extends AbstractEndpointSnitch - { - private final Map nodeToDc = new HashMap<>(); - - void add(InetAddressAndPort node, String dc) - { - nodeToDc.put(node, dc); - } - - public String getRack(InetAddressAndPort endpoint) - { - return null; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - return nodeToDc.get(endpoint); - } - - public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) - { - return 0; - } - } - @Test public void shouldCompressConnection_None() { DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.none); - Assert.assertFalse(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR)); + Assert.assertFalse(OutboundConnectionSettings.shouldCompressConnection(LOCAL_ADDR, REMOTE_ADDR)); } @Test public void shouldCompressConnection_DifferentDc() { - TestSnitch snitch = new TestSnitch(); - snitch.add(LOCAL_ADDR, "dc1"); - snitch.add(REMOTE_ADDR, "dc2"); - DatabaseDescriptor.setEndpointSnitch(snitch); + ClusterMetadataTestHelper.register(LOCAL_ADDR, DC1, RACK); + ClusterMetadataTestHelper.register(REMOTE_ADDR, DC2, RACK); DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.dc); - Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR)); + Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(LOCAL_ADDR, REMOTE_ADDR)); + } + + @Test + public void shouldCompressConnection_MetadataNotInitialized() + { + // if cluster metadata isn't yet available then we assume that every peer is remote. + // connections will be re-established once the cluster metadata service becomes available + ClusterMetadataTestHelper.register(LOCAL_ADDR, DC1, RACK); + ClusterMetadataTestHelper.register(REMOTE_ADDR, DC2, RACK); + DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.dc); + Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(LOCAL_ADDR, REMOTE_ADDR)); } @Test public void shouldCompressConnection_All() { DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.all); - Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR)); + Assert.assertTrue(OutboundConnectionSettings.shouldCompressConnection(LOCAL_ADDR, REMOTE_ADDR)); } @Test public void shouldCompressConnection_SameDc() { - TestSnitch snitch = new TestSnitch(); - snitch.add(LOCAL_ADDR, "dc1"); - snitch.add(REMOTE_ADDR, "dc1"); - DatabaseDescriptor.setEndpointSnitch(snitch); + ClusterMetadataTestHelper.register(LOCAL_ADDR, DC1, RACK); + ClusterMetadataTestHelper.register(REMOTE_ADDR, DC1, RACK); DatabaseDescriptor.setInternodeCompression(Config.InternodeCompression.dc); - Assert.assertFalse(OutboundConnectionSettings.shouldCompressConnection(getEndpointSnitch(), LOCAL_ADDR, REMOTE_ADDR)); + Assert.assertFalse(OutboundConnectionSettings.shouldCompressConnection(LOCAL_ADDR, REMOTE_ADDR)); } } diff --git a/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java b/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java index 5afbe88315..0f184ee8b7 100644 --- a/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java +++ b/test/unit/org/apache/cassandra/net/OutboundConnectionsTest.java @@ -41,6 +41,7 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.net.MessagingService.current_version; import static org.apache.cassandra.net.OutboundConnections.LARGE_MESSAGE_THRESHOLD; @@ -78,6 +79,11 @@ public class OutboundConnectionsTest { DatabaseDescriptor.daemonInitialization(); ClusterMetadataTestHelper.setInstanceForTest(); + // register the local broadcast address and REMOTE_ADDRESS addresses so the snitch is able to retrieve them + // from cluster metadata. This is necessary for the default of OutboundConnectionSettings::tcpNoDelay. + // There's no need to register RECONNECT_ADDRESS and it would be incorrect to do so + ClusterMetadataTestHelper.register(FBUtilities.getBroadcastAddressAndPort()); + ClusterMetadataTestHelper.register(REMOTE_ADDR); CommitLog.instance.start(); } diff --git a/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java b/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java index 47669e192e..6b49fafe0e 100644 --- a/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java +++ b/test/unit/org/apache/cassandra/net/ProxyHandlerConnectionsTest.java @@ -41,6 +41,7 @@ import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.io.IVersionedAsymmetricSerializer; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; @@ -87,6 +88,8 @@ public class ProxyHandlerConnectionsTest // call these to initialize everything in case a message is dropped, otherwise we will NPE in the commitlog CommitLog.instance.start(); ServerTestUtils.initCMS(); + // register the local broadcast address so the snitch can retrieve it when setting up connection defaults + ClusterMetadataTestHelper.register(FBUtilities.getBroadcastAddressAndPort()); CompactionManager.instance.getPendingTasks(); } diff --git a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java index b4aa22930f..ad5e8220e2 100644 --- a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java +++ b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java @@ -95,9 +95,9 @@ import org.apache.cassandra.gms.IGossiper; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.Locator; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.net.ConnectionType; import org.apache.cassandra.net.IVerbHandler; @@ -137,6 +137,7 @@ import org.apache.cassandra.streaming.StreamState; import org.apache.cassandra.streaming.StreamingChannel; import org.apache.cassandra.streaming.StreamingDataInputPlus; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tools.nodetool.Repair; import org.apache.cassandra.utils.AbstractTypeGenerators; import org.apache.cassandra.utils.CassandraGenerators; @@ -195,7 +196,6 @@ public abstract class FuzzTestBase extends CQLTester.InMemory DatabaseDescriptor.setCommitLogWriteDiskAccessMode(Config.DiskAccessMode.mmap); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); // TOOD (coverage): random select - DatabaseDescriptor.setLocalDataCenter("test"); StreamingChannel.Factory.Global.unsafeSet(new StreamingChannel.Factory() { private final AtomicInteger counter = new AtomicInteger(); @@ -674,7 +674,7 @@ public abstract class FuzzTestBase extends CQLTester.InMemory final Map nodes; private final IFailureDetector failureDetector = Mockito.mock(IFailureDetector.class); - private final IEndpointSnitch snitch = Mockito.mock(IEndpointSnitch.class); + private final Locator locator = Mockito.mock(Locator.class); private final SimulatedExecutorFactory globalExecutor; final ScheduledExecutorPlus unorderedScheduled; final ExecutorPlus orderedExecutor; @@ -732,8 +732,8 @@ public abstract class FuzzTestBase extends CQLTester.InMemory String dc = rs.pick(dcs); String rack = "rack"; - Mockito.when(snitch.getDatacenter(Mockito.eq(addressAndPort))).thenReturn(dc); - Mockito.when(snitch.getRack(Mockito.eq(addressAndPort))).thenReturn(rack); + Location location = new Location(dc, rack); + Mockito.when(locator.location(Mockito.eq(addressAndPort))).thenReturn(location); VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(DatabaseDescriptor.getPartitioner()); EndpointState state = new EndpointState(new HeartBeatState(42, 42)); @@ -1221,9 +1221,9 @@ public abstract class FuzzTestBase extends CQLTester.InMemory } @Override - public IEndpointSnitch snitch() + public Locator locator() { - return snitch; + return locator; } @Override diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index 42ed428c76..83e060fe32 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -45,8 +45,6 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.WithPartitioner; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -58,7 +56,6 @@ import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; @@ -87,32 +84,7 @@ public class StorageServiceServerTest GOSSIP_DISABLE_THREAD_VALIDATION.setBoolean(true); ServerTestUtils.daemonInitialization(); DatabaseDescriptor.setPartitionerUnsafe(OrderPreservingPartitioner.instance); - IEndpointSnitch snitch = new AbstractNetworkTopologySnitch() - { - @Override - public String getRack(InetAddressAndPort endpoint) - { - return location(endpoint).rack; - } - - @Override - public String getDatacenter(InetAddressAndPort endpoint) - { - return location(endpoint).datacenter; - } - - private Location location(InetAddressAndPort endpoint) - { - ClusterMetadata metadata = ClusterMetadata.current(); - NodeId id = metadata.directory.peerId(endpoint); - if (id == null) - throw new IllegalArgumentException("Unknown endpoint " + endpoint); - return metadata.directory.location(id); - } - }; ServerTestUtils.prepareServerNoRegister(); - DatabaseDescriptor.setEndpointSnitch(snitch); - id1 = InetAddressAndPort.getByName("127.0.0.1"); id2 = InetAddressAndPort.getByName("127.0.0.2"); id3 = InetAddressAndPort.getByName("127.0.0.3"); diff --git a/test/unit/org/apache/cassandra/service/StorageServiceTest.java b/test/unit/org/apache/cassandra/service/StorageServiceTest.java index 725f105db8..0b7ef7b1c4 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceTest.java @@ -30,15 +30,12 @@ import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.locator.AbstractEndpointSnitch; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.locator.ReplicaMultimap; -import org.apache.cassandra.locator.SimpleSnitch; +import org.apache.cassandra.locator.SimpleLocationProvider; import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.transformations.Register; @@ -73,27 +70,8 @@ public class StorageServiceTest extends TestBaseImpl DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); ClusterMetadataService.instance().commit(new Register(NodeAddresses.current(), - new Location(SimpleSnitch.DATA_CENTER_NAME, SimpleSnitch.RACK_NAME), + SimpleLocationProvider.LOCATION, NodeVersion.CURRENT)); - IEndpointSnitch snitch = new AbstractEndpointSnitch() - { - public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) - { - return 0; - } - - public String getRack(InetAddressAndPort endpoint) - { - return "R1"; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - return "DC1"; - } - }; - - DatabaseDescriptor.setEndpointSnitch(snitch); CommitLog.instance.start(); } @@ -298,7 +276,7 @@ public class StorageServiceTest extends TestBaseImpl { assertEquals(String.format("Provided datacenter '%s' is not a valid datacenter, available datacenters are: %s", nonExistentDC, - SimpleSnitch.DATA_CENTER_NAME), + SimpleLocationProvider.LOCATION.datacenter), ex.getMessage()); } } diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java index 6356f3ce1a..03785f3c30 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java @@ -36,13 +36,15 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.WriteType; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.locator.EndpointsForToken; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.NodeProximity; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.locator.ReplicaUtils; +import org.apache.cassandra.locator.BaseProximity; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.KeyspaceParams; @@ -81,26 +83,8 @@ public class WriteResponseHandlerTest SchemaLoader.loadSchema(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); // Register peers with expected DC for NetworkTopologyStrategy. -// metadata.clearUnsafe(); -// metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.255")); -// metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.2.0.255")); - - DatabaseDescriptor.setEndpointSnitch(new IEndpointSnitch() + NodeProximity sorter = new BaseProximity() { - public String getRack(InetAddressAndPort endpoint) - { - return null; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - byte[] address = endpoint.getAddress().getAddress(); - if (address[1] == 1) - return "datacenter1"; - else - return "datacenter2"; - } - public > C sortedByProximity(InetAddressAndPort address, C replicas) { return replicas; @@ -111,24 +95,22 @@ public class WriteResponseHandlerTest return 0; } - public void gossiperStarting() - { - - } - public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) { return false; } - }); + }; + DatabaseDescriptor.setNodeProximity(sorter); DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.1.0.1")); - SchemaLoader.createKeyspace("Foo", KeyspaceParams.nts("datacenter1", 3, "datacenter2", 3), SchemaLoader.standardCFMD("Foo", "Bar")); - ks = Keyspace.open("Foo"); - cfs = ks.getColumnFamilyStore("Bar"); targets = EndpointsForToken.of(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0)), full("127.1.0.255"), full("127.1.0.254"), full("127.1.0.253"), full("127.2.0.255"), full("127.2.0.254"), full("127.2.0.253")); + for (InetAddressAndPort ep : targets.endpoints()) + ClusterMetadataTestHelper.register(ep, ep.addressBytes[1] == 1 ? "datacenter1" : "datacenter2", "rack1"); pending = EndpointsForToken.empty(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0))); + SchemaLoader.createKeyspace("Foo", KeyspaceParams.nts("datacenter1", 3, "datacenter2", 3), SchemaLoader.standardCFMD("Foo", "Bar")); + ks = Keyspace.open("Foo"); + cfs = ks.getColumnFamilyStore("Bar"); } @Before diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java index 7eef9d7def..ed844343a4 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.service; -import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Set; import java.util.function.Predicate; @@ -36,10 +35,10 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.UnavailableException; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; -import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; @@ -93,49 +92,13 @@ public class WriteResponseHandlerTransientTest DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); // Register peers with expected DC for NetworkTopologyStrategy. - -// metadata.clearUnsafe(); -// metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.1")); -// metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.2.0.1")); - - DatabaseDescriptor.setEndpointSnitch(new IEndpointSnitch() - { - public String getRack(InetAddressAndPort endpoint) - { - return null; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - byte[] address = endpoint.getAddress().getAddress(); - if (address[1] == 1) - return DC1; - else - return DC2; - } - - public > C sortedByProximity(InetAddressAndPort address, C unsortedAddress) - { - return unsortedAddress; - } - - public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) - { - return 0; - } - - public void gossiperStarting() - { - - } - - public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) - { - return false; - } - }); - - DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.1.0.1")); + DatabaseDescriptor.setBroadcastAddress(EP1.getAddress()); + ClusterMetadataTestHelper.register(EP1, DC1, "r1"); + ClusterMetadataTestHelper.register(EP2, DC1, "r1"); + ClusterMetadataTestHelper.register(EP3, DC1, "r1"); + ClusterMetadataTestHelper.register(EP4, DC2, "r1"); + ClusterMetadataTestHelper.register(EP5, DC2, "r1"); + ClusterMetadataTestHelper.register(EP6, DC2, "r1"); SchemaLoader.createKeyspace("ks", KeyspaceParams.nts(DC1, "3/1", DC2, "3/1"), SchemaLoader.standardCFMD("ks", "tbl")); ks = Keyspace.open("ks"); cfs = ks.getColumnFamilyStore("tbl"); diff --git a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java index fdafc9e388..046da259e9 100644 --- a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java @@ -22,8 +22,10 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.locator.ReplicaPlan; import org.junit.Before; import org.junit.BeforeClass; @@ -76,6 +78,8 @@ public class ReadExecutorTest full(InetAddressAndPort.getByName("127.0.0.254")), full(InetAddressAndPort.getByName("127.0.0.253")) ); + for (InetAddressAndPort ep : targets.endpoints()) + ClusterMetadataTestHelper.register(ep, ServerTestUtils.DATA_CENTER, ServerTestUtils.RACK1); cfs.sampleReadLatencyMicros = 0; } diff --git a/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java b/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java index 8d4fee6d18..aaa88c938d 100644 --- a/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java +++ b/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.service.reads.range; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -27,15 +28,20 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.membership.Location; import static org.apache.cassandra.ServerTestUtils.markCMS; import static org.apache.cassandra.ServerTestUtils.recreateCMS; @@ -65,14 +71,18 @@ public class ReplicaPlanMergerTest private static Keyspace keyspace; @BeforeClass - public static void defineSchema() throws ConfigurationException + public static void defineSchema() throws ConfigurationException, UnknownHostException { ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true); - SchemaLoader.prepareServer(); + ServerTestUtils.prepareServerNoRegister(); StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); recreateCMS(); SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(2)); keyspace = Keyspace.open(KEYSPACE); + // ensure all endpoints used in the tests are located in the same DC + Location local = DatabaseDescriptor.getLocator().local(); + for (int i = 1; i <= 3; i++) + ClusterMetadataTestHelper.register(InetAddressAndPort.getByName("127.0.0." + i), local); markCMS(); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java index b2ff6723ee..f0026d35b9 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java @@ -60,7 +60,6 @@ import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; @@ -227,23 +226,6 @@ public abstract class AbstractReadRepairTest { ServerTestUtils.prepareServerNoRegister(); - DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch() - { - public String getRack(InetAddressAndPort endpoint) - { - return "rack1"; - } - - public String getDatacenter(InetAddressAndPort endpoint) - { - byte[] address = endpoint.addressBytes; - if (address[1] == 2) { - return "datacenter2"; - } - return "datacenter1"; - } - }); - target1 = InetAddressAndPort.getByName("127.1.0.255"); target2 = InetAddressAndPort.getByName("127.1.0.254"); target3 = InetAddressAndPort.getByName("127.1.0.253"); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java index e3de74be15..5138de0300 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import com.google.common.collect.Iterables; import org.apache.cassandra.Util; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.EndpointsForRange; @@ -56,6 +57,7 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.ByteBufferUtil; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -138,6 +140,9 @@ public class ReadRepairTest cell2 = cell("v", "val2", now); cell3 = cell("v", "val3", now); resolved = mutation(cell1, cell2); + Location local = DatabaseDescriptor.getLocator().local(); + for (InetAddressAndPort endpoint : targets.endpoints()) + ClusterMetadataTestHelper.register(endpoint, local); } private static DecoratedKey dk(int v) diff --git a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java index d25d02c80d..bd91832d00 100644 --- a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java +++ b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java @@ -24,10 +24,8 @@ import java.util.Random; import java.util.concurrent.ThreadLocalRandom; import com.google.common.collect.Iterables; -import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; @@ -68,12 +66,6 @@ import static org.junit.Assert.assertTrue; public class ClusterMetadataTransformationTest { - @BeforeClass - public static void init() - { - ServerTestUtils.initSnitch(); - } - long seed = System.nanoTime(); Random random = new Random(seed); diff --git a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java index 3ec661f158..167aa3992b 100644 --- a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java @@ -79,7 +79,7 @@ public class GetLogStateTest ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(cms); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); } @Test diff --git a/test/unit/org/apache/cassandra/tcm/LogStateTest.java b/test/unit/org/apache/cassandra/tcm/LogStateTest.java index 9388ad6333..187337f1c9 100644 --- a/test/unit/org/apache/cassandra/tcm/LogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/LogStateTest.java @@ -70,7 +70,7 @@ public class LogStateTest ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(cms); log.readyUnchecked(); - log.bootstrap(FBUtilities.getBroadcastAddressAndPort()); + log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); } @Test diff --git a/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java b/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java index 28fffd9619..f29e61ce82 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/RingTest.java @@ -25,7 +25,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.locator.SimpleSnitch; +import org.apache.cassandra.locator.SimpleLocationProvider; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tools.ToolRunner; import org.apache.cassandra.utils.FBUtilities; @@ -76,11 +76,11 @@ public class RingTest extends CQLTester */ String[] lines = tool.getStdout().split("\\R"); - assertThat(lines[1].trim()).endsWith(SimpleSnitch.DATA_CENTER_NAME); + assertThat(lines[1].trim()).endsWith(SimpleLocationProvider.LOCATION.datacenter); assertThat(lines[3]).containsPattern("Address *Rack *Status *State *Load *Owns *Token *"); String hostRing = lines[lines.length-4].trim(); // this command has a couple extra newlines and an empty error message at the end. Not messing with it. assertThat(hostRing).startsWith(hostForm); - assertThat(hostRing).contains(SimpleSnitch.RACK_NAME); + assertThat(hostRing).contains(SimpleLocationProvider.LOCATION.rack); assertThat(hostRing).contains("Up"); assertThat(hostRing).contains("Normal"); assertThat(hostRing).containsPattern("\\d+\\.?\\d+ KiB"); diff --git a/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java index 9d8496c8b1..cc9c14deb4 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/StatusTest.java @@ -24,7 +24,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.locator.SimpleSnitch; +import org.apache.cassandra.locator.SimpleLocationProvider; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tools.ToolRunner; @@ -86,7 +86,7 @@ public class StatusTest extends CQLTester assertThat(hostStatus).containsPattern("\\d+\\.?\\d+ KiB"); assertThat(hostStatus).contains(localHostId); assertThat(hostStatus).contains(token); - assertThat(hostStatus).endsWith(SimpleSnitch.RACK_NAME); + assertThat(hostStatus).endsWith(SimpleLocationProvider.LOCATION.rack); String bootstrappingWarn = lines[lines.length-1].trim(); assertThat(bootstrappingWarn) @@ -106,7 +106,7 @@ public class StatusTest extends CQLTester UN localhost 45.71 KiB 100.0% 0b1b5e91-ad3b-444e-9c24-50578486978a 1849950853373272258 rack1 */ String[] lines = PATTERN.split(tool.getStdout()); - assertThat(lines[0].trim()).endsWith(SimpleSnitch.DATA_CENTER_NAME); + assertThat(lines[0].trim()).endsWith(SimpleLocationProvider.LOCATION.datacenter); String hostStatus = lines[lines.length-1].trim(); assertThat(hostStatus).startsWith("UN"); assertThat(hostStatus).contains(hostForm); @@ -114,7 +114,7 @@ public class StatusTest extends CQLTester assertThat(hostStatus).containsPattern("\\d+\\.\\d+%"); assertThat(hostStatus).contains(localHostId); assertThat(hostStatus).contains(token); - assertThat(hostStatus).endsWith(SimpleSnitch.RACK_NAME); + assertThat(hostStatus).endsWith(SimpleLocationProvider.LOCATION.rack); assertThat(hostStatus).doesNotContain("?"); } }