Speed up nodetool doc generation by producing all command help in a single jvm

patch by Maxim Muzafarov, reviewed by Dmitry Konstantinov for CASSANDRA-21444

CHANGES.txt

@@ -1,4 +1,5 @@
 6.0-alpha2
+ * Speed up nodetool doc generation by producing all command help in a single jvm (CASSANDRA-21444)
  * Always send TCM commit failures as Messaging failures (CASSANDRA-21457)
  * Fix ReadCommand serializedSize() using incorrect epoch (CASSANDRA-21438)
  * Allocation improvements in ProtocolVersion, StorageProxy and MerkleTree (CASSANDRA-21199)

build.xml

@@ -605,7 +605,29 @@
         </wikitext-to-html>
     </target>
 
-    <target name="gen-asciidoc" description="Generate dynamic asciidoc pages" depends="jar" unless="ant.gen-doc.skip">
+    <target name="gen-nodetool-text" description="Generate nodetool command help text files consumed by the docs" depends="jar" unless="ant.gen-doc.skip">
+        <property name="nodetool.help.outdir" location="${doc.dir}/modules/cassandra/examples/TEXT/NODETOOL"/>
+        <property name="nodetool.help.classes" location="${build.dir}/nodetool-helpgen"/>
+        <mkdir dir="${nodetool.help.outdir}"/>
+        <mkdir dir="${nodetool.help.classes}"/>
+        <javac srcdir="${test.unit.src}" includes="org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java"
+               destdir="${nodetool.help.classes}" includeantruntime="false" classpathref="cassandra.classpath"/>
+        <java classname="org.apache.cassandra.tools.nodetool.NodetoolHelpGenerator" fork="true" failonerror="true">
+            <classpath>
+                <pathelement location="${nodetool.help.classes}"/>
+                <pathelement location="${basedir}/conf"/>
+                <path refid="cassandra.classpath"/>
+            </classpath>
+            <jvmarg line="${java-jvmargs}"/>
+            <sysproperty key="logback.configurationFile" value="logback-tools.xml"/>
+            <sysproperty key="cassandra.cli.layout" value="airline"/>
+            <arg value="--dir"/>
+            <arg value="${nodetool.help.outdir}"/>
+            <arg value="--txt"/>
+        </java>
+    </target>
+
+    <target name="gen-asciidoc" description="Generate dynamic asciidoc pages" depends="jar,gen-nodetool-text" unless="ant.gen-doc.skip">
         <exec executable="make" osfamily="unix" dir="${doc.dir}" failonerror="true">
             <arg value="gen-asciidoc"/>
         </exec>

doc/scripts/gen-nodetool-docs.py

@@ -14,84 +14,53 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-A script to use nodetool to generate documentation for nodetool
+A script to generate the nodetool AsciiDoc pages from the help text files
+produced by the 'gen-nodetool-text' Ant target.
 """
 from __future__ import print_function
 
 import os
 import re
 import sys
-import subprocess
-from subprocess import PIPE
-from subprocess import Popen
-from itertools import islice
-from threading import Thread
-
-batch_size = 3
 
 if(os.environ.get("SKIP_NODETOOL") == "1"):
     sys.exit(0)
 
 
-nodetool = "../bin/nodetool"
 outdir = "modules/cassandra/pages/managing/tools/nodetool"
 examplesdir = "modules/cassandra/examples/TEXT/NODETOOL"
 helpfilename = outdir + "/nodetool.txt"
 command_re = re.compile("(    )([_a-z]+)")
 commandADOCContent = "= {0}\n\n== Usage\n[source,plaintext]\n----\ninclude::cassandra:example$TEXT/NODETOOL/{0}.txt[]\n----\n"
 
-# https://docs.python.org/3/library/itertools.html#itertools-recipes
-def batched(iterable, n):
-    "Batch data into tuples of length n. The last batch may be shorter."
-    # batched('ABCDEFG', 3) --> ABC DEF G
-    if n < 1:
-        raise ValueError('n must be at least one')
-    it = iter(iterable)
-    batch = tuple(islice(it, n))
-    while (batch):
-        yield batch
-        batch = tuple(islice(it, n))
-
 # create the documentation directory
 if not os.path.exists(outdir):
     os.makedirs(outdir)
 
 # create the base help file to use for discovering the commands
 def create_help_file():
+    rootfile = examplesdir + "/nodetool.txt"
+    if not os.path.exists(rootfile):
+        sys.exit("ERROR: '%s' is missing. Run 'ant gen-nodetool-text' (or 'ant gen-asciidoc') first." % rootfile)
+    with open(rootfile, "r") as f:
+        out = f.read()
     with open(helpfilename, "w+") as output_file:
-        try:
+        # Wrap the initial "usage: ..." block
-            proc = subprocess.Popen([nodetool, "help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        usage_block = re.search(r'usage:.*?\n\n', out, re.DOTALL | re.MULTILINE)
-            out, err = proc.communicate()
+        if usage_block:
-            if proc.returncode != 0:
+            output_file.write("[source,console]\n----\n")
-                print(
+            output_file.write(usage_block.group(0))
-                    'ERROR: Nodetool failed to run, you likely need to build '
+            output_file.write("----\n")
-                    'cassandra using ant jar from the top level directory'
+            out = out.replace(usage_block.group(0), '')
-                )
+        else:
-                raise subprocess.CalledProcessError(proc.returncode, proc.args, output=out, stderr=err)
+            raise ValueError("No usage block matched in nodetool help output")
+        output_file.write(out)
 
-            # Wrap the initial "usage: ..." block
+# for a given command, create the ADOC file to contain it
-            usage_block = re.search(r'usage:.*?\n\n', out.decode('utf-8'), re.DOTALL | re.MULTILINE)
-            if usage_block:
-                output_file.write("[source,console]\n----\n")
-                output_file.write(usage_block.group(0))
-                output_file.write("----\n")
-                out = out.decode('utf-8').replace(usage_block.group(0), '')
-            else:
-                raise ValueError("No usage block matched in nodetool help output")
-            output_file.write(out)
-        except subprocess.CalledProcessError as cpe:
-            raise cpe
-
-# for a given command, create the help file and an ADOC file to contain it
 def create_adoc(command):
     if command:
         cmdName = command.group(0).strip()
-        cmdFilename = examplesdir + "/" + cmdName + ".txt"
         adocFilename = outdir + "/" + cmdName + ".adoc"
-        with open(cmdFilename, "wb+") as cmdFile:
-            proc = Popen([nodetool, "help", cmdName], stdin=PIPE, stdout=PIPE)
-            (out, err) = proc.communicate()
-            cmdFile.write(out)
         with open(adocFilename, "w+") as adocFile:
             adocFile.write(commandADOCContent.format(cmdName,cmdName,cmdName))
 
@@ -108,12 +77,5 @@ with open(outdir + "/nodetool.adoc", "w+") as output:
 
 # create the command usage pages
 with open(helpfilename, "r+") as helpfile:
-    for clis in batched(helpfile, batch_size):
+    for commandLine in helpfile:
-        threads = []
+        create_adoc(command_re.match(commandLine))
-        for commandLine in clis:
-            command = command_re.match(commandLine)
-            t = Thread(target=create_adoc, args=[command])
-            threads.append(t)
-            t.start()
-        for t in threads:
-            t.join()

test/unit/org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java

@@ -18,25 +18,26 @@
 
 package org.apache.cassandra.tools.nodetool;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.tools.INodeProbeFactory;
-import org.apache.cassandra.tools.ToolRunner;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+import org.apache.cassandra.tools.Output;
 
 import static com.google.common.collect.Lists.newArrayList;
 
@@ -45,47 +46,78 @@ import static com.google.common.collect.Lists.newArrayList;
  * a separator between command hierarchy levels in the file names (e.g. {@code "info$threads"}) due to the fact that
  * a command name can contain special characters like {@code -} or {@code _}.
  * <p>
- * The generator calls the {@code ./nodetool help} command to get the list of available commands and their descriptions,
+ * The help output is produced in-process (single JVM); be sure to run the generator after the jars are built
- * in order to generate the latest help output for each command be sure to run the generator after the jars are built
+ * (e.g. {@code ant jar}). Pass {@code --dir <path>} to override the output directory and {@code --txt} to append
- * (e.g. {@code ant jar}).
+ * a {@code .txt} extension (used by {@code doc/scripts/gen-nodetool-docs.py}).
  */
 public class NodetoolHelpGenerator
 {
     private static final Logger logger = LoggerFactory.getLogger(NodetoolHelpGenerator.class);
-    private static final Map<String, String> ENV = ImmutableMap.of("JAVA_HOME", CassandraRelevantProperties.JAVA_HOME.getString());
     private static final String NODETOOL_COMMAND_HELP_WRITE_DIR = "test/resources/nodetool/help/";
+    private static final String ROOT_COMMAND_FILE = "nodetool";
     private static final String IGNORE_LINE = "        With no arguments,";
     private static final String NODETOOL_COMMAND_LIST_START_AFTER = "The most commonly used nodetool commands are:";
     private static final String NODETOOL_SUBCOMMAND_LIST_START_AFTER = "COMMANDS";
     private static final Pattern NODETOOL_COMMAND_DESCRIPTION_SPACES = Pattern.compile("^ {4}(\\S+)");
     private static final Pattern NODETOOL_SUBCOMMAND_DESCRIPTION_SPACES = Pattern.compile("^ {8}(\\S+)");
     private static final String COMMAND_FULL_NAME_SEPARATOR = "$";
+    private static final INodeProbeFactory NO_PROBE = new INodeProbeFactory()
+    {
+        public NodeProbe create(String host, int port) { throw new UnsupportedOperationException(); }
+        public NodeProbe create(String host, int port, String user, String pass) { throw new UnsupportedOperationException(); }
+    };
+
+    private final String writeDir;
+    private final String extension;
+
+    public NodetoolHelpGenerator(String writeDir, String extension)
+    {
+        this.writeDir = writeDir;
+        this.extension = extension;
+    }
 
     /**
-     * Main method to generate help files for all nodetool commands to the {@code test/resources/nodetool/help/}.
+     * Main method to generate help files for all nodetool commands to the {@code test/resources/nodetool/help/}
+     * (or to the directory specified via {@code --dir}).
      * <p>
      * For example, the {@code nodetool help bootstrap resume} help output results in a file
      * {@code test/resources/nodetool/help/bootstrap$resume}, where the {@code $} character
-     * is used as a separator for the subcommand. The arguments are passed as a list of commands
+     * is used as a separator for the subcommand. Trailing positional arguments generate the help
-     * to generate help files for. For example, {@code bootstrap resume} is passed.
+     * for a single command only (e.g. {@code bootstrap resume}).
-     * <p>
-     * By default, the files are written to {@code test/resources/nodetool/help/}.
      */
     public static void main(String[] args)
     {
-        List<String> commands = new ArrayList<>(List.of(args));
+        String dir = NODETOOL_COMMAND_HELP_WRITE_DIR;
-//        commands.add("assassinate");
+        String extension = "";
+        List<String> commands = new ArrayList<>();
+        for (int i = 0; i < args.length; i++)
+        {
+            switch (args[i])
+            {
+                case "--dir":
+                    if (++i >= args.length)
+                        throw new IllegalArgumentException("--dir requires a path");
+                    dir = args[i];
+                    break;
+                case "--txt":
+                    extension = ".txt";
+                    break;
+                default: commands.add(args[i]);
+            }
+        }
 
+        NodetoolHelpGenerator generator = new NodetoolHelpGenerator(dir, extension);
         if (commands.isEmpty())
-            new NodetoolHelpGenerator().writeCommandsHelpOutput();
+            generator.writeCommandsHelpOutput();
         else
-            new NodetoolHelpGenerator().writer(commands);
+            generator.writer(commands);
     }
 
     public void writeCommandsHelpOutput()
     {
-        List<String> roots = find(() -> ToolRunner.invoke(ENV, newArrayList("bin/nodetool", "help")),
+        List<String> roots = find(() -> help(new ArrayList<>()),
                                   NODETOOL_COMMAND_LIST_START_AFTER, NODETOOL_COMMAND_DESCRIPTION_SPACES);
+        writer(new ArrayList<>());
 
         for (String command : roots)
             writeToFileRecursively(newArrayList(command), this::writer);
@@ -93,9 +125,7 @@ public class NodetoolHelpGenerator
 
     private void writeToFileRecursively(List<String> hierarchy, Consumer<List<String>> writer)
     {
-        List<String> subcommands = find(() -> ToolRunner.invoke(ENV, Lists.asList("bin/nodetool",
+        List<String> subcommands = find(() -> help(hierarchy),
-                                                                                  "help",
-                                                                                  hierarchy.toArray(new String[0]))),
                                         NODETOOL_SUBCOMMAND_LIST_START_AFTER, NODETOOL_SUBCOMMAND_DESCRIPTION_SPACES);
         for (String subcommand : subcommands)
         {
@@ -109,13 +139,12 @@ public class NodetoolHelpGenerator
 
     public void writer(List<String> fullCommand)
     {
-        ToolRunner.ToolResult result = ToolRunner.invoke(ENV, Lists.asList("bin/nodetool", "help",
+        String stdout = help(fullCommand);
-                                                                           fullCommand.toArray(new String[0])));
+        String name = fullCommand.isEmpty() ? ROOT_COMMAND_FILE : String.join(COMMAND_FULL_NAME_SEPARATOR, fullCommand);
-        result.assertOnCleanExit();
 
         try
         {
-            File commandHelpOut = new File(NODETOOL_COMMAND_HELP_WRITE_DIR, String.join(COMMAND_FULL_NAME_SEPARATOR, fullCommand)); //checkstyle: permit this instantiation
+            File commandHelpOut = new File(writeDir, name + extension); //checkstyle: permit this instantiation
             boolean created = commandHelpOut.getParentFile().mkdirs();
             if (created)
                 logger.debug("Created directory: {}", commandHelpOut.getParentFile().getAbsolutePath());
@@ -126,7 +155,7 @@ public class NodetoolHelpGenerator
 
             try (FileWriter fw = new FileWriter(commandHelpOut))
             {
-                fw.write(result.getStdout().trim());
+                fw.write(stdout.trim());
                 fw.write("\n");
             }
             logger.info("The help is written for '{}' to '{}'", fullCommand, commandHelpOut.getAbsolutePath());
@@ -137,11 +166,31 @@ public class NodetoolHelpGenerator
         }
     }
 
-    private static List<String> find(Supplier<ToolRunner.ToolResult> cmdResult, String afterLine, Pattern commandPattern)
+    private static String help(List<String> command)
     {
-        ToolRunner.ToolResult result = cmdResult.get();
+        List<String> args = new ArrayList<>();
-        result.assertOnCleanExit();
+        if (!command.isEmpty())
-        String[] lines = result.getStdout().split("\n");
+        {
+            args.add("help");
+            args.addAll(command);
+        }
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        ByteArrayOutputStream err = new ByteArrayOutputStream();
+        PrintStream outStream = new PrintStream(out, true, StandardCharsets.UTF_8);
+        PrintStream errStream = new PrintStream(err, true, StandardCharsets.UTF_8);
+        int rc = new NodeTool(NO_PROBE, new Output(outStream, errStream)).execute(args.toArray(new String[0]));
+        outStream.flush();
+        errStream.flush();
+        String stderr = err.toString(StandardCharsets.UTF_8);
+        if (rc != 0 || !stderr.trim().isEmpty())
+            throw new RuntimeException("nodetool help " + String.join(" ", command) + " failed (rc=" + rc + "): "
+                                       + stderr);
+        return out.toString(StandardCharsets.UTF_8);
+    }
+
+    private static List<String> find(Supplier<String> stdout, String afterLine, Pattern commandPattern)
+    {
+        String[] lines = stdout.get().split("\n");
         List<String> commands = new ArrayList<>();
         boolean start = false;
         for (String line : lines)