diff --git a/CHANGES.txt b/CHANGES.txt index 5ab33a229b..a1213090e2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1 + * Tool to list data paths of existing tables (CASSANDRA-17568) * Migrate track_warnings to more standard naming conventions and use latest configuration types rather than long (CASSANDRA-17560) * Add support for CONTAINS and CONTAINS KEY in conditional UPDATE and DELETE statement (CASSANDRA-10537) * Migrate advanced config parameters to the new Config types (CASSANDRA-17431) diff --git a/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc b/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc index f80d039695..a313432cbb 100644 --- a/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc +++ b/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc @@ -240,3 +240,46 @@ concurrent compactions such that compactions complete quickly but don't take too many resources away from query threads is very important for performance. If you notice compaction unable to keep up, try tuning Cassandra's `concurrent_compactors` or `compaction_throughput` options. + +[[nodetool-datapaths]] +== Paths used for data files + +Cassandra is persisting data on disk within the configured directories. Data +files are distributed among the directories configured with `data_file_directories`. +Resembling the structure of keyspaces and tables, Cassandra is creating +subdirectories within `data_file_directories`. However, directories aren't removed +even if the tables and keyspaces are dropped. While these directories are kept with +the reason of holding snapshots, they are subject to removal. This is where operators +need to know which directories are still in use. Running the `nodetool datapaths` +command is an easy way to list in which directories Cassandra is actually storing +sstable data on disk. + +[source, bash] +---- +% nodetool datapaths -- system_auth +Keyspace: system_auth + Table: role_permissions + Paths: + /var/lib/cassandra/data/system_auth/role_permissions-3afbe79f219431a7add7f5ab90d8ec9c + + Table: network_permissions + Paths: + /var/lib/cassandra/data/system_auth/network_permissions-d46780c22f1c3db9b4c1b8d9fbc0cc23 + + Table: resource_role_permissons_index + Paths: + /var/lib/cassandra/data/system_auth/resource_role_permissons_index-5f2fbdad91f13946bd25d5da3a5c35ec + + Table: roles + Paths: + /var/lib/cassandra/data/system_auth/roles-5bc52802de2535edaeab188eecebb090 + + Table: role_members + Paths: + /var/lib/cassandra/data/system_auth/role_members-0ecdaa87f8fb3e6088d174fb36fe5c0d + +---- + +By default all keyspaces and tables are listed, however, a list of `keyspace` and +`keyspace.table` arguments can be given to query specific data paths. Using the `--format` +option the output can be formatted as YAML or JSON. diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index f9422bdbba..476353fee0 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -102,6 +102,7 @@ public class NodeTool Compact.class, CompactionHistory.class, CompactionStats.class, + DataPaths.class, Decommission.class, DescribeCluster.class, DescribeRing.class, diff --git a/src/java/org/apache/cassandra/tools/nodetool/DataPaths.java b/src/java/org/apache/cassandra/tools/nodetool/DataPaths.java new file mode 100644 index 0000000000..10ae01e8da --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/DataPaths.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool; + +import java.util.ArrayList; +import java.util.List; + +import io.airlift.airline.Arguments; +import io.airlift.airline.Command; +import io.airlift.airline.Option; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool.NodeToolCmd; +import org.apache.cassandra.tools.nodetool.stats.DataPathsHolder; +import org.apache.cassandra.tools.nodetool.stats.DataPathsPrinter; +import org.apache.cassandra.tools.nodetool.stats.StatsPrinter; + +@Command(name = "datapaths", description = "Print all directories where data of tables are stored") +public class DataPaths extends NodeToolCmd +{ + @Arguments(usage = "[...]", description = "List of table (or keyspace) names") + private List tableNames = new ArrayList<>(); + + @Option(title = "format", name = {"-F", "--format"}, description = "Output format (json, yaml)") + private String outputFormat = ""; + + @Override + protected void execute(NodeProbe probe) + { + if (!outputFormat.isEmpty() && !"json".equals(outputFormat) && !"yaml".equals(outputFormat)) + { + throw new IllegalArgumentException("arguments for -F are yaml and json only."); + } + + DataPathsHolder holder = new DataPathsHolder(probe, tableNames); + StatsPrinter printer = DataPathsPrinter.from(outputFormat); + printer.print(holder, probe.output().out); + } +} diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java new file mode 100644 index 0000000000..9bf7bdb45f --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool.stats; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Throwables.getStackTraceAsString; + +import org.apache.cassandra.db.ColumnFamilyStoreMBean; +import org.apache.cassandra.tools.NodeProbe; + +public class DataPathsHolder implements StatsHolder +{ + public final Map pathsHash = new HashMap<>(); + + public DataPathsHolder(NodeProbe probe, List tableNames) + { + Iterator> mbeansIterator = probe.getColumnFamilyStoreMBeanProxies(); + while (mbeansIterator.hasNext()) + { + Map.Entry entry = mbeansIterator.next(); + String keyspaceName = entry.getKey(); + String tableName = entry.getValue().getTableName(); + + if (!(tableNames.isEmpty() || + tableNames.contains(keyspaceName + '.' + tableName) || + tableNames.contains(keyspaceName) )) + { + continue; + } + + Map> ksPaths; + List dataPaths; + + try + { + dataPaths = entry.getValue().getDataPaths(); + } + catch (Throwable e) + { + probe.output().err.println("Failed to get data paths for " + keyspaceName + '.' + tableName + ". Skipped."); + probe.output().err.println("error: " + e.getMessage()); + probe.output().err.println("-- StackTrace --"); + probe.output().err.println(getStackTraceAsString(e)); + continue; + } + + if (pathsHash.containsKey(keyspaceName)) + { + ksPaths = (Map>) pathsHash.get(keyspaceName); + } + else + { + ksPaths = new HashMap<>(); + pathsHash.put(keyspaceName, ksPaths); + } + ksPaths.put(tableName, dataPaths); + } + } + + @Override + public Map convert2Map() + { + return pathsHash; + } +} diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java new file mode 100644 index 0000000000..ba2b29e766 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool.stats; + +import java.io.PrintStream; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class DataPathsPrinter +{ + public static StatsPrinter from(String format) + { + if ("json".equals(format)) + return new StatsPrinter.JsonPrinter<>(); + if ("yaml".equals(format)) + return new StatsPrinter.YamlPrinter<>(); + + return new DefaultPrinter(); + } + + public static class DefaultPrinter implements StatsPrinter + { + @Override + public void print(DataPathsHolder data, PrintStream out) + { + Iterator> iterator = data.pathsHash.entrySet().iterator(); + + while (iterator.hasNext()) + { + Map.Entry entry = iterator.next(); + + out.println("Keyspace: " + entry.getKey()); + Map> ksPaths = (Map>) entry.getValue(); + for (Map.Entry> table : ksPaths.entrySet()) + { + out.println("\tTable: " + table.getKey()); + out.println("\tPaths:"); + + for (String path : table.getValue()) + out.println("\t\t" + path); + + out.println(""); + } + } + } + } +} diff --git a/test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java new file mode 100644 index 0000000000..d09aceb3af --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool; + +import org.apache.commons.lang3.StringUtils; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.tools.ToolRunner; +import static org.assertj.core.api.Assertions.assertThat; + +public class DataPathsTest extends CQLTester +{ + private static final String SUBCOMMAND = "datapaths"; + + @BeforeClass + public static void setup() throws Exception + { + requireNetwork(); + startJMXServer(); + } + + @Test + public void testHelp() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("help", SUBCOMMAND); + tool.assertOnCleanExit(); + + String help = "NAME\n" + + " nodetool datapaths - Print all directories where data of tables are\n" + + " stored\n" + + '\n' + + "SYNOPSIS\n" + + " nodetool [(-h | --host )] [(-p | --port )]\n" + + " [(-pp | --print-port)] [(-pw | --password )]\n" + + " [(-pwf | --password-file )]\n" + + " [(-u | --username )] datapaths\n" + + " [(-F | --format )] [--] [...]\n" + + '\n' + + "OPTIONS\n" + + " -F , --format \n" + + " Output format (json, yaml)\n" + + '\n' + + " -h , --host \n" + + " Node hostname or ip address\n" + + '\n' + + " -p , --port \n" + + " Remote jmx agent port number\n" + + '\n' + + " -pp, --print-port\n" + + " Operate in 4.0 mode with hosts disambiguated by port number\n" + + '\n' + + " -pw , --password \n" + + " Remote jmx agent password\n" + + '\n' + + " -pwf , --password-file \n" + + " Path to the JMX password file\n" + + '\n' + + " -u , --username \n" + + " Remote jmx agent username\n" + + '\n' + + " --\n" + + " This option can be used to separate command-line options from the\n" + + " list of argument, (useful when arguments might be mistaken for\n" + + " command-line options\n" + + '\n' + + " [...]\n" + + " List of table (or keyspace) names\n" + + '\n' + + '\n'; + assertThat(tool.getStdout()).isEqualTo(help); + } + + @Test + public void testAllOutput() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND); + tool.assertOnCleanExit(); + assertThat(tool.getStdout()).contains("Keyspace: system_schema"); + assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isGreaterThan(1); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isGreaterThan(1); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isGreaterThan(1); + } + + @Test + public void testSelectedKeyspace() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_traces"); + tool.assertOnCleanExit(); + assertThat(tool.getStdout()).contains("Keyspace: system_traces"); + assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(1); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isGreaterThan(1); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isGreaterThan(1); + } + + @Test + public void testSelectedMultipleKeyspaces() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_traces", "system_auth"); + tool.assertOnCleanExit(); + assertThat(tool.getStdout()).contains("Keyspace: system_traces"); + assertThat(tool.getStdout()).contains("Keyspace: system_auth"); + assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(2); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isGreaterThan(1); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isGreaterThan(1); + } + + @Test + public void testSelectedTable() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_auth.roles"); + tool.assertOnCleanExit(); + assertThat(tool.getStdout()).contains("Keyspace: system_auth"); + assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(1); + assertThat(tool.getStdout()).contains("Table: roles"); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isEqualTo(1); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isEqualTo(1); + } + + @Test + public void testSelectedMultipleTables() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "system_auth.roles", "system_auth.role_members"); + tool.assertOnCleanExit(); + assertThat(tool.getStdout()).contains("Keyspace: system_auth"); + assertThat(StringUtils.countMatches(tool.getStdout(), "Keyspace:")).isEqualTo(1); + assertThat(tool.getStdout()).contains("Table: roles"); + assertThat(tool.getStdout()).contains("Table: role_members"); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tTable:")).isEqualTo(2); + assertThat(StringUtils.countMatches(tool.getStdout(), "\tPaths:")).isEqualTo(2); + } + + @Test + public void testFormatArgJson() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "--format", "json"); + tool.assertOnCleanExit(); + } + + @Test + public void testFormatArgYaml() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "--format", "yaml"); + tool.assertOnCleanExit(); + } + + @Test + public void testFormatArgBad() + { + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND, "--format", "bad"); + assertThat(tool.getStdout()).contains("arguments for -F are yaml and json only."); + } +}