natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add `allocate_tokens_for_local_rf` yaml option for token allocation that doesn't require keyspace knowledge/existence 

patch by Mick Semb Wever; reviewed by Branimir Lambov for CASSANDRA-15260
This commit is contained in:
Mick Semb Wever 2019-08-05 00:06:30 +02:00
parent fc4381ca89
commit 068d2d37c6
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
9 changed files with 105 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -1,3 +1,6 @@
4.0-alpha2
* Add `allocate_tokens_for_local_replication_factor` option for token allocation (CASSANDRA-15260)
4.0-alpha1
* Inaccurate exception message with nodetool snapshot (CASSANDRA-15287)
* Fix InternodeOutboundMetrics overloaded bytes/count mixup (CASSANDRA-15186)

10
conf/cassandra.yaml
View File

@ -26,15 +26,21 @@ num_tokens: 256
# Triggers automatic allocation of num_tokens tokens for this node. The allocation
# algorithm attempts to choose tokens in a way that optimizes replicated load over
# the nodes in the datacenter for the replication strategy used by the specified
# keyspace.
# the nodes in the datacenter for the replica factor.
#
# The load assigned to each node will be close to proportional to its number of
# vnodes.
#
# Only supported with the Murmur3Partitioner.
# Replica factor is determined via the replication strategy used by the specified
# keyspace.
# allocate_tokens_for_keyspace: KEYSPACE
# Replica factor is explicitly set, regardless of keyspace or datacenter.
# This is the replica factor within the datacenter, like NTS.
# allocate_tokens_for_local_replication_factor: 3
# initial_token allows you to specify tokens manually. While you can use it with
# vnodes (num_tokens > 1, above) -- in which case you should provide a
# comma-separated list -- it's primarily used when adding nodes to legacy clusters

2
src/java/org/apache/cassandra/config/Config.java
View File

@ -86,6 +86,8 @@ public class Config
public int num_tokens = 1;
/** Triggers automatic allocation of tokens if set, using the replication strategy of the referenced keyspace */
public String allocate_tokens_for_keyspace = null;
/** Triggers automatic allocation of tokens if set, based on the provided replica count for a datacenter */
public Integer allocate_tokens_for_local_replication_factor = null;
public long native_transport_idle_timeout_in_ms = 0L;

5
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -1422,6 +1422,11 @@ public class DatabaseDescriptor
return System.getProperty(Config.PROPERTY_PREFIX + "allocate_tokens_for_keyspace", conf.allocate_tokens_for_keyspace);
}
public static Integer getAllocateTokensForLocalRf()
{
return conf.allocate_tokens_for_local_replication_factor;
}
public static Collection<String> tokensFromString(String tokenString)
{
List<String> tokens = new ArrayList<String>();

22
src/java/org/apache/cassandra/dht/BootStrapper.java
View File

@ -24,6 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.db.Keyspace;
@ -32,6 +33,7 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.*;
@ -153,6 +155,7 @@ public class BootStrapper extends ProgressEventNotifierSupport
public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddressAndPort address, int schemaWaitDelay) throws ConfigurationException
{
String allocationKeyspace = DatabaseDescriptor.getAllocateTokensForKeyspace();
Integer allocationLocalRf = DatabaseDescriptor.getAllocateTokensForLocalRf();
Collection<String> initialTokens = DatabaseDescriptor.getInitialTokens();
if (initialTokens.size() > 0 && allocationKeyspace != null)
logger.warn("manually specified tokens override automatic allocation");
@ -172,6 +175,9 @@ public class BootStrapper extends ProgressEventNotifierSupport
if (allocationKeyspace != null)
return allocateTokens(metadata, address, allocationKeyspace, numTokens, schemaWaitDelay);
if (allocationLocalRf != null)
return allocateTokens(metadata, address, allocationLocalRf, numTokens, schemaWaitDelay);
if (numTokens == 1)
logger.warn("Picking random token for a single vnode. You should probably add more vnodes and/or use the automatic token allocation mechanism.");
@ -215,6 +221,22 @@ public class BootStrapper extends ProgressEventNotifierSupport
return tokens;
}
static Collection<Token> allocateTokens(final TokenMetadata metadata,
InetAddressAndPort address,
int rf,
int numTokens,
int schemaWaitDelay)
{
StorageService.instance.waitForSchema(schemaWaitDelay);
if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
Gossiper.waitToSettle();
Collection<Token> tokens = TokenAllocation.allocateTokens(metadata, rf, address, numTokens);
BootstrapDiagnostics.tokensAllocated(address, metadata, rf, numTokens, tokens);
return tokens;
}
public static Collection<Token> getRandomTokens(TokenMetadata metadata, int numTokens)
{
Set<Token> tokens = new HashSet<>(numTokens);

16
src/java/org/apache/cassandra/dht/BootstrapDiagnostics.java
View File

@ -45,6 +45,7 @@ final class BootstrapDiagnostics
address,
null,
allocationKeyspace,
null,
numTokens,
ImmutableList.copyOf(initialTokens)));
}
@ -56,6 +57,7 @@ final class BootstrapDiagnostics
address,
metadata.cloneOnlyTokenMap(),
null,
null,
numTokens,
ImmutableList.copyOf(tokens)));
}
@ -68,6 +70,20 @@ final class BootstrapDiagnostics
address,
metadata.cloneOnlyTokenMap(),
allocationKeyspace,
null,
numTokens,
ImmutableList.copyOf(tokens)));
}
static void tokensAllocated(InetAddressAndPort address, TokenMetadata metadata,
int rf, int numTokens, Collection<Token> tokens)
{
if (isEnabled(BootstrapEventType.TOKENS_ALLOCATED))
service.publish(new BootstrapEvent(BootstrapEventType.TOKENS_ALLOCATED,
address,
metadata.cloneOnlyTokenMap(),
null,
rf,
numTokens,
ImmutableList.copyOf(tokens)));
}

6
src/java/org/apache/cassandra/dht/BootstrapEvent.java
View File

@ -42,16 +42,19 @@ final class BootstrapEvent extends DiagnosticEvent
private final InetAddressAndPort address;
@Nullable
private final String allocationKeyspace;
@Nullable
private final Integer rf;
private final Integer numTokens;
private final Collection<Token> tokens;
BootstrapEvent(BootstrapEventType type, InetAddressAndPort address, @Nullable TokenMetadata tokenMetadata,
@Nullable String allocationKeyspace, int numTokens, ImmutableCollection<Token> tokens)
@Nullable String allocationKeyspace, @Nullable Integer rf, int numTokens, ImmutableCollection<Token> tokens)
{
this.type = type;
this.address = address;
this.tokenMetadata = tokenMetadata;
this.allocationKeyspace = allocationKeyspace;
this.rf = rf;
this.numTokens = numTokens;
this.tokens = tokens;
}
@ -75,6 +78,7 @@ final class BootstrapEvent extends DiagnosticEvent
HashMap<String, Serializable> ret = new HashMap<>();
ret.put("tokenMetadata", String.valueOf(tokenMetadata));
ret.put("allocationKeyspace", allocationKeyspace);
ret.put("rf", rf);
ret.put("numTokens", numTokens);
ret.put("tokens", String.valueOf(tokens));
return ret;

25
src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
View File

@ -32,6 +32,7 @@ import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
@ -72,6 +73,20 @@ public class TokenAllocation
return tokens;
}
public static Collection<Token> allocateTokens(final TokenMetadata tokenMetadata,
final int replicas,
final InetAddressAndPort endpoint,
int numTokens)
{
TokenMetadata tokenMetadataCopy = tokenMetadata.cloneOnlyTokenMap();
StrategyAdapter strategy = getStrategy(tokenMetadataCopy, replicas, endpoint);
Collection<Token> tokens = create(tokenMetadata, strategy).addUnit(endpoint, numTokens);
tokens = adjustForCrossDatacenterClashes(tokenMetadata, strategy, tokens);
logger.warn("Selected tokens {}", tokens);
// SummaryStatistics is not implemented for `allocate_tokens_for_local_replication_factor`
return tokens;
}
private static Collection<Token> adjustForCrossDatacenterClashes(final TokenMetadata tokenMetadata,
StrategyAdapter strategy, Collection<Token> tokens)
{
@ -197,7 +212,17 @@ public class TokenAllocation
{
final String dc = snitch.getDatacenter(endpoint);
final int replicas = rs.getReplicationFactor(dc).allReplicas;
return getStrategy(tokenMetadata, replicas, snitch, endpoint);
}
static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final int replicas, final InetAddressAndPort endpoint)
{
return getStrategy(tokenMetadata, replicas, DatabaseDescriptor.getEndpointSnitch(), endpoint);
}
static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final int replicas, final IEndpointSnitch snitch, final InetAddressAndPort endpoint)
{
final String dc = snitch.getDatacenter(endpoint);
if (replicas == 0 || replicas == 1)
{
// No replication, each node is treated as separate.

19
test/unit/org/apache/cassandra/dht/BootStrapperTest.java
View File

@ -174,6 +174,17 @@ public class BootStrapperTest
allocateTokensForNode(vn, ks, tm, addr);
}
@Test
public void testAllocateTokensLocalRf() throws UnknownHostException
{
int vn = 16;
int allocateTokensForLocalRf = 3;
TokenMetadata tm = new TokenMetadata();
generateFakeEndpoints(tm, 10, vn);
InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
allocateTokensForNode(vn, allocateTokensForLocalRf, tm, addr);
}
public void testAllocateTokensNetworkStrategy(int rackCount, int replicas) throws UnknownHostException
{
IEndpointSnitch oldSnitch = DatabaseDescriptor.getEndpointSnitch();
@ -243,6 +254,14 @@ public class BootStrapperTest
verifyImprovement(os, ns);
}
private void allocateTokensForNode(int vn, int rf, TokenMetadata tm, InetAddressAndPort addr)
{
Collection<Token> tokens = BootStrapper.allocateTokens(tm, addr, rf, vn, 0);
assertEquals(vn, tokens.size());
tm.updateNormalTokens(tokens, addr);
// SummaryStatistics is not implemented for `allocate_tokens_for_local_replication_factor` so can't be verified
}
private void verifyImprovement(SummaryStatistics os, SummaryStatistics ns)
{
if (ns.getStandardDeviation() > os.getStandardDeviation())