diff --git a/CHANGES.txt b/CHANGES.txt index 3f1ab2cebe..cc92b39a05 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ * Allow nodetool garbagecollect to take a user defined list of SSTables (CASSANDRA-16767) * Add a guardrail for misprepared statements (CASSANDRA-21139) Merged from 6.0: + * Speed up nodetool doc generation by producing all command help in a single jvm (CASSANDRA-21444) * Always send TCM commit failures as Messaging failures (CASSANDRA-21457) * Fix ReadCommand serializedSize() using incorrect epoch (CASSANDRA-21438) * Allocation improvements in ProtocolVersion, StorageProxy and MerkleTree (CASSANDRA-21199) diff --git a/build.xml b/build.xml index ecdfa14cd7..cb8ee95990 100644 --- a/build.xml +++ b/build.xml @@ -605,7 +605,29 @@ - + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/scripts/gen-nodetool-docs.py b/doc/scripts/gen-nodetool-docs.py index c4eb109662..6accfd4589 100644 --- a/doc/scripts/gen-nodetool-docs.py +++ b/doc/scripts/gen-nodetool-docs.py @@ -14,84 +14,53 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -A script to use nodetool to generate documentation for nodetool +A script to generate the nodetool AsciiDoc pages from the help text files +produced by the 'gen-nodetool-text' Ant target. """ from __future__ import print_function import os import re import sys -import subprocess -from subprocess import PIPE -from subprocess import Popen -from itertools import islice -from threading import Thread - -batch_size = 3 if(os.environ.get("SKIP_NODETOOL") == "1"): sys.exit(0) -nodetool = "../bin/nodetool" outdir = "modules/cassandra/pages/managing/tools/nodetool" examplesdir = "modules/cassandra/examples/TEXT/NODETOOL" helpfilename = outdir + "/nodetool.txt" command_re = re.compile("( )([_a-z]+)") commandADOCContent = "= {0}\n\n== Usage\n[source,plaintext]\n----\ninclude::cassandra:example$TEXT/NODETOOL/{0}.txt[]\n----\n" -# https://docs.python.org/3/library/itertools.html#itertools-recipes -def batched(iterable, n): - "Batch data into tuples of length n. The last batch may be shorter." - # batched('ABCDEFG', 3) --> ABC DEF G - if n < 1: - raise ValueError('n must be at least one') - it = iter(iterable) - batch = tuple(islice(it, n)) - while (batch): - yield batch - batch = tuple(islice(it, n)) - # create the documentation directory if not os.path.exists(outdir): os.makedirs(outdir) # create the base help file to use for discovering the commands def create_help_file(): + rootfile = examplesdir + "/nodetool.txt" + if not os.path.exists(rootfile): + sys.exit("ERROR: '%s' is missing. Run 'ant gen-nodetool-text' (or 'ant gen-asciidoc') first." % rootfile) + with open(rootfile, "r") as f: + out = f.read() with open(helpfilename, "w+") as output_file: - try: - proc = subprocess.Popen([nodetool, "help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - print( - 'ERROR: Nodetool failed to run, you likely need to build ' - 'cassandra using ant jar from the top level directory' - ) - raise subprocess.CalledProcessError(proc.returncode, proc.args, output=out, stderr=err) + # Wrap the initial "usage: ..." block + usage_block = re.search(r'usage:.*?\n\n', out, re.DOTALL | re.MULTILINE) + if usage_block: + output_file.write("[source,console]\n----\n") + output_file.write(usage_block.group(0)) + output_file.write("----\n") + out = out.replace(usage_block.group(0), '') + else: + raise ValueError("No usage block matched in nodetool help output") + output_file.write(out) - # Wrap the initial "usage: ..." block - usage_block = re.search(r'usage:.*?\n\n', out.decode('utf-8'), re.DOTALL | re.MULTILINE) - if usage_block: - output_file.write("[source,console]\n----\n") - output_file.write(usage_block.group(0)) - output_file.write("----\n") - out = out.decode('utf-8').replace(usage_block.group(0), '') - else: - raise ValueError("No usage block matched in nodetool help output") - output_file.write(out) - except subprocess.CalledProcessError as cpe: - raise cpe - -# for a given command, create the help file and an ADOC file to contain it +# for a given command, create the ADOC file to contain it def create_adoc(command): if command: cmdName = command.group(0).strip() - cmdFilename = examplesdir + "/" + cmdName + ".txt" adocFilename = outdir + "/" + cmdName + ".adoc" - with open(cmdFilename, "wb+") as cmdFile: - proc = Popen([nodetool, "help", cmdName], stdin=PIPE, stdout=PIPE) - (out, err) = proc.communicate() - cmdFile.write(out) with open(adocFilename, "w+") as adocFile: adocFile.write(commandADOCContent.format(cmdName,cmdName,cmdName)) @@ -108,12 +77,5 @@ with open(outdir + "/nodetool.adoc", "w+") as output: # create the command usage pages with open(helpfilename, "r+") as helpfile: - for clis in batched(helpfile, batch_size): - threads = [] - for commandLine in clis: - command = command_re.match(commandLine) - t = Thread(target=create_adoc, args=[command]) - threads.append(t) - t.start() - for t in threads: - t.join() + for commandLine in helpfile: + create_adoc(command_re.match(commandLine)) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java b/test/unit/org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java index e899cda01d..ba788a1831 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java @@ -18,25 +18,26 @@ package org.apache.cassandra.tools.nodetool; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.tools.ToolRunner; +import org.apache.cassandra.tools.INodeProbeFactory; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool; +import org.apache.cassandra.tools.Output; import static com.google.common.collect.Lists.newArrayList; @@ -45,47 +46,78 @@ import static com.google.common.collect.Lists.newArrayList; * a separator between command hierarchy levels in the file names (e.g. {@code "info$threads"}) due to the fact that * a command name can contain special characters like {@code -} or {@code _}. *

- * The generator calls the {@code ./nodetool help} command to get the list of available commands and their descriptions, - * in order to generate the latest help output for each command be sure to run the generator after the jars are built - * (e.g. {@code ant jar}). + * The help output is produced in-process (single JVM); be sure to run the generator after the jars are built + * (e.g. {@code ant jar}). Pass {@code --dir } to override the output directory and {@code --txt} to append + * a {@code .txt} extension (used by {@code doc/scripts/gen-nodetool-docs.py}). */ public class NodetoolHelpGenerator { private static final Logger logger = LoggerFactory.getLogger(NodetoolHelpGenerator.class); - private static final Map ENV = ImmutableMap.of("JAVA_HOME", CassandraRelevantProperties.JAVA_HOME.getString()); private static final String NODETOOL_COMMAND_HELP_WRITE_DIR = "test/resources/nodetool/help/"; + private static final String ROOT_COMMAND_FILE = "nodetool"; private static final String IGNORE_LINE = " With no arguments,"; private static final String NODETOOL_COMMAND_LIST_START_AFTER = "The most commonly used nodetool commands are:"; private static final String NODETOOL_SUBCOMMAND_LIST_START_AFTER = "COMMANDS"; private static final Pattern NODETOOL_COMMAND_DESCRIPTION_SPACES = Pattern.compile("^ {4}(\\S+)"); private static final Pattern NODETOOL_SUBCOMMAND_DESCRIPTION_SPACES = Pattern.compile("^ {8}(\\S+)"); private static final String COMMAND_FULL_NAME_SEPARATOR = "$"; + private static final INodeProbeFactory NO_PROBE = new INodeProbeFactory() + { + public NodeProbe create(String host, int port) { throw new UnsupportedOperationException(); } + public NodeProbe create(String host, int port, String user, String pass) { throw new UnsupportedOperationException(); } + }; + + private final String writeDir; + private final String extension; + + public NodetoolHelpGenerator(String writeDir, String extension) + { + this.writeDir = writeDir; + this.extension = extension; + } /** - * Main method to generate help files for all nodetool commands to the {@code test/resources/nodetool/help/}. + * Main method to generate help files for all nodetool commands to the {@code test/resources/nodetool/help/} + * (or to the directory specified via {@code --dir}). *

* For example, the {@code nodetool help bootstrap resume} help output results in a file * {@code test/resources/nodetool/help/bootstrap$resume}, where the {@code $} character - * is used as a separator for the subcommand. The arguments are passed as a list of commands - * to generate help files for. For example, {@code bootstrap resume} is passed. - *

- * By default, the files are written to {@code test/resources/nodetool/help/}. + * is used as a separator for the subcommand. Trailing positional arguments generate the help + * for a single command only (e.g. {@code bootstrap resume}). */ public static void main(String[] args) { - List commands = new ArrayList<>(List.of(args)); -// commands.add("assassinate"); + String dir = NODETOOL_COMMAND_HELP_WRITE_DIR; + String extension = ""; + List commands = new ArrayList<>(); + for (int i = 0; i < args.length; i++) + { + switch (args[i]) + { + case "--dir": + if (++i >= args.length) + throw new IllegalArgumentException("--dir requires a path"); + dir = args[i]; + break; + case "--txt": + extension = ".txt"; + break; + default: commands.add(args[i]); + } + } + NodetoolHelpGenerator generator = new NodetoolHelpGenerator(dir, extension); if (commands.isEmpty()) - new NodetoolHelpGenerator().writeCommandsHelpOutput(); + generator.writeCommandsHelpOutput(); else - new NodetoolHelpGenerator().writer(commands); + generator.writer(commands); } public void writeCommandsHelpOutput() { - List roots = find(() -> ToolRunner.invoke(ENV, newArrayList("bin/nodetool", "help")), + List roots = find(() -> help(new ArrayList<>()), NODETOOL_COMMAND_LIST_START_AFTER, NODETOOL_COMMAND_DESCRIPTION_SPACES); + writer(new ArrayList<>()); for (String command : roots) writeToFileRecursively(newArrayList(command), this::writer); @@ -93,9 +125,7 @@ public class NodetoolHelpGenerator private void writeToFileRecursively(List hierarchy, Consumer> writer) { - List subcommands = find(() -> ToolRunner.invoke(ENV, Lists.asList("bin/nodetool", - "help", - hierarchy.toArray(new String[0]))), + List subcommands = find(() -> help(hierarchy), NODETOOL_SUBCOMMAND_LIST_START_AFTER, NODETOOL_SUBCOMMAND_DESCRIPTION_SPACES); for (String subcommand : subcommands) { @@ -109,13 +139,12 @@ public class NodetoolHelpGenerator public void writer(List fullCommand) { - ToolRunner.ToolResult result = ToolRunner.invoke(ENV, Lists.asList("bin/nodetool", "help", - fullCommand.toArray(new String[0]))); - result.assertOnCleanExit(); + String stdout = help(fullCommand); + String name = fullCommand.isEmpty() ? ROOT_COMMAND_FILE : String.join(COMMAND_FULL_NAME_SEPARATOR, fullCommand); try { - File commandHelpOut = new File(NODETOOL_COMMAND_HELP_WRITE_DIR, String.join(COMMAND_FULL_NAME_SEPARATOR, fullCommand)); //checkstyle: permit this instantiation + File commandHelpOut = new File(writeDir, name + extension); //checkstyle: permit this instantiation boolean created = commandHelpOut.getParentFile().mkdirs(); if (created) logger.debug("Created directory: {}", commandHelpOut.getParentFile().getAbsolutePath()); @@ -126,7 +155,7 @@ public class NodetoolHelpGenerator try (FileWriter fw = new FileWriter(commandHelpOut)) { - fw.write(result.getStdout().trim()); + fw.write(stdout.trim()); fw.write("\n"); } logger.info("The help is written for '{}' to '{}'", fullCommand, commandHelpOut.getAbsolutePath()); @@ -137,11 +166,31 @@ public class NodetoolHelpGenerator } } - private static List find(Supplier cmdResult, String afterLine, Pattern commandPattern) + private static String help(List command) { - ToolRunner.ToolResult result = cmdResult.get(); - result.assertOnCleanExit(); - String[] lines = result.getStdout().split("\n"); + List args = new ArrayList<>(); + if (!command.isEmpty()) + { + args.add("help"); + args.addAll(command); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + PrintStream outStream = new PrintStream(out, true, StandardCharsets.UTF_8); + PrintStream errStream = new PrintStream(err, true, StandardCharsets.UTF_8); + int rc = new NodeTool(NO_PROBE, new Output(outStream, errStream)).execute(args.toArray(new String[0])); + outStream.flush(); + errStream.flush(); + String stderr = err.toString(StandardCharsets.UTF_8); + if (rc != 0 || !stderr.trim().isEmpty()) + throw new RuntimeException("nodetool help " + String.join(" ", command) + " failed (rc=" + rc + "): " + + stderr); + return out.toString(StandardCharsets.UTF_8); + } + + private static List find(Supplier stdout, String afterLine, Pattern commandPattern) + { + String[] lines = stdout.get().split("\n"); List commands = new ArrayList<>(); boolean start = false; for (String line : lines)