natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add datapaths subcommand to nodetool 

patch by Tibor Repasi; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-17568
This commit is contained in:
Tibor Répási 2022-04-20 22:10:13 +02:00 committed by Stefan Miklosovic
parent 1ad8bf67a9
commit c26dc06a28
7 changed files with 415 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.1
* Tool to list data paths of existing tables (CASSANDRA-17568)
* Migrate track_warnings to more standard naming conventions and use latest configuration types rather than long (CASSANDRA-17560)
* Add support for CONTAINS and CONTAINS KEY in conditional UPDATE and DELETE statement (CASSANDRA-10537)
* Migrate advanced config parameters to the new Config types (CASSANDRA-17431)

43
doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc
View File

@ -240,3 +240,46 @@ concurrent compactions such that compactions complete quickly but don't
take too many resources away from query threads is very important for
performance. If you notice compaction unable to keep up, try tuning
Cassandra's `concurrent_compactors` or `compaction_throughput` options.
[[nodetool-datapaths]]
== Paths used for data files
Cassandra is persisting data on disk within the configured directories. Data
files are distributed among the directories configured with `data_file_directories`.
Resembling the structure of keyspaces and tables, Cassandra is creating
subdirectories within `data_file_directories`. However, directories aren't removed
even if the tables and keyspaces are dropped. While these directories are kept with
the reason of holding snapshots, they are subject to removal. This is where operators
need to know which directories are still in use. Running the `nodetool datapaths`
command is an easy way to list in which directories Cassandra is actually storing
sstable data on disk.
[source, bash]
----
% nodetool datapaths -- system_auth
Keyspace: system_auth
Table: role_permissions
Paths:
/var/lib/cassandra/data/system_auth/role_permissions-3afbe79f219431a7add7f5ab90d8ec9c
Table: network_permissions
Paths:
/var/lib/cassandra/data/system_auth/network_permissions-d46780c22f1c3db9b4c1b8d9fbc0cc23
Table: resource_role_permissons_index
Paths:
/var/lib/cassandra/data/system_auth/resource_role_permissons_index-5f2fbdad91f13946bd25d5da3a5c35ec
Table: roles
Paths:
/var/lib/cassandra/data/system_auth/roles-5bc52802de2535edaeab188eecebb090
Table: role_members
Paths:
/var/lib/cassandra/data/system_auth/role_members-0ecdaa87f8fb3e6088d174fb36fe5c0d
----
By default all keyspaces and tables are listed, however, a list of `keyspace` and
`keyspace.table` arguments can be given to query specific data paths. Using the `--format`
option the output can be formatted as YAML or JSON.

1
src/java/org/apache/cassandra/tools/NodeTool.java
View File

@ -102,6 +102,7 @@ public class NodeTool
Compact.class,
CompactionHistory.class,
CompactionStats.class,
DataPaths.class,
Decommission.class,
DescribeCluster.class,
DescribeRing.class,

53
src/java/org/apache/cassandra/tools/nodetool/DataPaths.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools.nodetool;
import java.util.ArrayList;
import java.util.List;
import io.airlift.airline.Arguments;
import io.airlift.airline.Command;
import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
import org.apache.cassandra.tools.nodetool.stats.DataPathsHolder;
import org.apache.cassandra.tools.nodetool.stats.DataPathsPrinter;
import org.apache.cassandra.tools.nodetool.stats.StatsPrinter;
@Command(name = "datapaths", description = "Print all directories where data of tables are stored")
public class DataPaths extends NodeToolCmd
{
@Arguments(usage = "[<keyspace.table>...]", description = "List of table (or keyspace) names")
private List<String> tableNames = new ArrayList<>();
@Option(title = "format", name = {"-F", "--format"}, description = "Output format (json, yaml)")
private String outputFormat = "";
@Override
protected void execute(NodeProbe probe)
{
if (!outputFormat.isEmpty() && !"json".equals(outputFormat) && !"yaml".equals(outputFormat))
{
throw new IllegalArgumentException("arguments for -F are yaml and json only.");
}
DataPathsHolder holder = new DataPathsHolder(probe, tableNames);
StatsPrinter<DataPathsHolder> printer = DataPathsPrinter.from(outputFormat);
printer.print(holder, probe.output().out);
}
}

84
src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools.nodetool.stats;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static com.google.common.base.Throwables.getStackTraceAsString;
import org.apache.cassandra.db.ColumnFamilyStoreMBean;
import org.apache.cassandra.tools.NodeProbe;
public class DataPathsHolder implements StatsHolder
{
public final Map<String, Object> pathsHash = new HashMap<>();
public DataPathsHolder(NodeProbe probe, List<String> tableNames)
{
Iterator<Map.Entry<String, ColumnFamilyStoreMBean>> mbeansIterator = probe.getColumnFamilyStoreMBeanProxies();
while (mbeansIterator.hasNext())
{
Map.Entry<String, ColumnFamilyStoreMBean> entry = mbeansIterator.next();
String keyspaceName = entry.getKey();
String tableName = entry.getValue().getTableName();
if (!(tableNames.isEmpty() ||
tableNames.contains(keyspaceName + '.' + tableName) ||
tableNames.contains(keyspaceName) ))
{
continue;
}
Map<String, List<String>> ksPaths;
List<String> dataPaths;
try
{
dataPaths = entry.getValue().getDataPaths();
}
catch (Throwable e)
{
probe.output().err.println("Failed to get data paths for " + keyspaceName + '.' + tableName + ". Skipped.");
probe.output().err.println("error: " + e.getMessage());
probe.output().err.println("-- StackTrace --");
probe.output().err.println(getStackTraceAsString(e));
continue;
}
if (pathsHash.containsKey(keyspaceName))
{
ksPaths = (Map<String, List<String>>) pathsHash.get(keyspaceName);
}
else
{
ksPaths = new HashMap<>();
pathsHash.put(keyspaceName, ksPaths);
}
ksPaths.put(tableName, dataPaths);
}
}
@Override
public Map<String, Object> convert2Map()
{
return pathsHash;
}
}

63
src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools.nodetool.stats;
import java.io.PrintStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public class DataPathsPrinter<T extends StatsHolder>
{
public static StatsPrinter<DataPathsHolder> from(String format)
{
if ("json".equals(format))
return new StatsPrinter.JsonPrinter<>();
if ("yaml".equals(format))
return new StatsPrinter.YamlPrinter<>();
return new DefaultPrinter();
}
public static class DefaultPrinter implements StatsPrinter<DataPathsHolder>
{
@Override
public void print(DataPathsHolder data, PrintStream out)
{
Iterator<Map.Entry<String, Object>> iterator = data.pathsHash.entrySet().iterator();
while (iterator.hasNext())
{
Map.Entry<String, Object> entry = iterator.next();
out.println("Keyspace: " + entry.getKey());
Map<String, List<String>> ksPaths = (Map<String, List<String>>) entry.getValue();
for (Map.Entry<String, List<String>> table : ksPaths.entrySet())
{
out.println("\tTable: " + table.getKey());
out.println("\tPaths:");
for (String path : table.getValue())
out.println("\t\t" + path);
out.println("");
}
}
}
}
}

170
test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java Normal file
View File

@ -0,0 +1,170 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools.nodetool;
import org.apache.commons.lang3.StringUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.tools.ToolRunner;
import static org.assertj.core.api.Assertions.assertThat;
public class DataPathsTest extends CQLTester
{
private static final String SUBCOMMAND = "datapaths";
@BeforeClass
public static void setup() throws Exception
{
requireNetwork();
startJMXServer();
}
@Test
public void testHelp()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("help", SUBCOMMAND);
tool.assertOnCleanExit();
String help = "NAME\n" +
" nodetool datapaths - Print all directories where data of tables are\n" +
" stored\n" +
'\n' +
"SYNOPSIS\n" +
" nodetool [(-h <host> | --host <host>)] [(-p <port> | --port <port>)]\n" +
" [(-pp | --print-port)] [(-pw <password> | --password <password>)]\n" +
" [(-pwf <passwordFilePath> | --password-file <passwordFilePath>)]\n" +
" [(-u <username> | --username <username>)] datapaths\n" +
" [(-F <format> | --format <format>)] [--] [<keyspace.table>...]\n" +
'\n' +
"OPTIONS\n" +
" -F <format>, --format <format>\n" +
" Output format (json, yaml)\n" +
'\n' +
" -h <host>, --host <host>\n" +
" Node hostname or ip address\n" +
'\n' +
" -p <port>, --port <port>\n" +
" Remote jmx agent port number\n" +
'\n' +
" -pp, --print-port\n" +
" Operate in 4.0 mode with hosts disambiguated by port number\n" +
'\n' +
" -pw <password>, --password <password>\n" +
" Remote jmx agent password\n" +
'\n' +
" -pwf <passwordFilePath>, --password-file <passwordFilePath>\n" +
" Path to the JMX password file\n" +
'\n' +
" -u <username>, --username <username>\n" +
" Remote jmx agent username\n" +
'\n' +
" --\n" +
" This option can be used to separate command-line options from the\n" +
" list of argument, (useful when arguments might be mistaken for\n" +
" command-line options\n" +
'\n' +
" [<keyspace.table>...]\n" +
" List of table (or keyspace) names\n" +
'\n' +
'\n';
assertThat(tool.getStdout()).isEqualTo(help);
}
@Test
public void testAllOutput()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND);
tool.assertOnCleanExit();
assertThat(tool.getStdout()).contains("Keyspace: system_schema");
assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isGreaterThan(1);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isGreaterThan(1);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isGreaterThan(1);
}
@Test
public void testSelectedKeyspace()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_traces");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).contains("Keyspace: system_traces");
assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(1);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isGreaterThan(1);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isGreaterThan(1);
}
@Test
public void testSelectedMultipleKeyspaces()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_traces", "system_auth");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).contains("Keyspace: system_traces");
assertThat(tool.getStdout()).contains("Keyspace: system_auth");
assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(2);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isGreaterThan(1);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isGreaterThan(1);
}
@Test
public void testSelectedTable()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_auth.roles");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).contains("Keyspace: system_auth");
assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(1);
assertThat(tool.getStdout()).contains("Table: roles");
assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isEqualTo(1);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isEqualTo(1);
}
@Test
public void testSelectedMultipleTables()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_auth.roles", "system_auth.role_members");
tool.assertOnCleanExit();
assertThat(tool.getStdout()).contains("Keyspace: system_auth");
assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(1);
assertThat(tool.getStdout()).contains("Table: roles");
assertThat(tool.getStdout()).contains("Table: role_members");
assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isEqualTo(2);
assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isEqualTo(2);
}
@Test
public void testFormatArgJson()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "--format", "json");
tool.assertOnCleanExit();
}
@Test
public void testFormatArgYaml()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "--format", "yaml");
tool.assertOnCleanExit();
}
@Test
public void testFormatArgBad()
{
ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "--format", "bad");
assertThat(tool.getStdout()).contains("arguments for -F are yaml and json only.");
}
}