natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Speed up nodetool doc generation by producing all command help in a single jvm 

patch by Maxim Muzafarov, reviewed by Dmitry Konstantinov for CASSANDRA-21444
This commit is contained in:
Maxim Muzafarov 2026-06-07 20:39:51 +02:00
parent 3d40c8825c
commit 7899b594b8
No known key found for this signature in database
GPG Key ID: 7FEC714D84388C16
4 changed files with 125 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
6.0-alpha2
* Speed up nodetool doc generation by producing all command help in a single jvm (CASSANDRA-21444)
* Always send TCM commit failures as Messaging failures (CASSANDRA-21457)
* Fix ReadCommand serializedSize() using incorrect epoch (CASSANDRA-21438)
* Allocation improvements in ProtocolVersion, StorageProxy and MerkleTree (CASSANDRA-21199)

24
build.xml
View File

@ -605,7 +605,29 @@
</wikitext-to-html>
</target>
<target name="gen-asciidoc" description="Generate dynamic asciidoc pages" depends="jar" unless="ant.gen-doc.skip">
<target name="gen-nodetool-text" description="Generate nodetool command help text files consumed by the docs" depends="jar" unless="ant.gen-doc.skip">
<property name="nodetool.help.outdir" location="${doc.dir}/modules/cassandra/examples/TEXT/NODETOOL"/>
<property name="nodetool.help.classes" location="${build.dir}/nodetool-helpgen"/>
<mkdir dir="${nodetool.help.outdir}"/>
<mkdir dir="${nodetool.help.classes}"/>
<javac srcdir="${test.unit.src}" includes="org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java"
destdir="${nodetool.help.classes}" includeantruntime="false" classpathref="cassandra.classpath"/>
<java classname="org.apache.cassandra.tools.nodetool.NodetoolHelpGenerator" fork="true" failonerror="true">
<classpath>
<pathelement location="${nodetool.help.classes}"/>
<pathelement location="${basedir}/conf"/>
<path refid="cassandra.classpath"/>
</classpath>
<jvmarg line="${java-jvmargs}"/>
<sysproperty key="logback.configurationFile" value="logback-tools.xml"/>
<sysproperty key="cassandra.cli.layout" value="airline"/>
<arg value="--dir"/>
<arg value="${nodetool.help.outdir}"/>
<arg value="--txt"/>
</java>
</target>
<target name="gen-asciidoc" description="Generate dynamic asciidoc pages" depends="jar,gen-nodetool-text" unless="ant.gen-doc.skip">
<exec executable="make" osfamily="unix" dir="${doc.dir}" failonerror="true">
<arg value="gen-asciidoc"/>
</exec>

78
doc/scripts/gen-nodetool-docs.py
View File

@ -14,84 +14,53 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
A script to use nodetool to generate documentation for nodetool
A script to generate the nodetool AsciiDoc pages from the help text files
produced by the 'gen-nodetool-text' Ant target.
"""
from __future__ import print_function
import os
import re
import sys
import subprocess
from subprocess import PIPE
from subprocess import Popen
from itertools import islice
from threading import Thread
batch_size = 3
if(os.environ.get("SKIP_NODETOOL") == "1"):
sys.exit(0)
nodetool = "../bin/nodetool"
outdir = "modules/cassandra/pages/managing/tools/nodetool"
examplesdir = "modules/cassandra/examples/TEXT/NODETOOL"
helpfilename = outdir + "/nodetool.txt"
command_re = re.compile("( )([_a-z]+)")
commandADOCContent = "= {0}\n\n== Usage\n[source,plaintext]\n----\ninclude::cassandra:example$TEXT/NODETOOL/{0}.txt[]\n----\n"
# https://docs.python.org/3/library/itertools.html#itertools-recipes
def batched(iterable, n):
"Batch data into tuples of length n. The last batch may be shorter."
# batched('ABCDEFG', 3) --> ABC DEF G
if n < 1:
raise ValueError('n must be at least one')
it = iter(iterable)
batch = tuple(islice(it, n))
while (batch):
yield batch
batch = tuple(islice(it, n))
# create the documentation directory
if not os.path.exists(outdir):
os.makedirs(outdir)
# create the base help file to use for discovering the commands
def create_help_file():
rootfile = examplesdir + "/nodetool.txt"
if not os.path.exists(rootfile):
sys.exit("ERROR: '%s' is missing. Run 'ant gen-nodetool-text' (or 'ant gen-asciidoc') first." % rootfile)
with open(rootfile, "r") as f:
out = f.read()
with open(helpfilename, "w+") as output_file:
try:
proc = subprocess.Popen([nodetool, "help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
if proc.returncode != 0:
print(
'ERROR: Nodetool failed to run, you likely need to build '
'cassandra using ant jar from the top level directory'
)
raise subprocess.CalledProcessError(proc.returncode, proc.args, output=out, stderr=err)
# Wrap the initial "usage: ..." block
usage_block = re.search(r'usage:.*?\n\n', out, re.DOTALL | re.MULTILINE)
if usage_block:
output_file.write("[source,console]\n----\n")
output_file.write(usage_block.group(0))
output_file.write("----\n")
out = out.replace(usage_block.group(0), '')
else:
raise ValueError("No usage block matched in nodetool help output")
output_file.write(out)
# Wrap the initial "usage: ..." block
usage_block = re.search(r'usage:.*?\n\n', out.decode('utf-8'), re.DOTALL | re.MULTILINE)
if usage_block:
output_file.write("[source,console]\n----\n")
output_file.write(usage_block.group(0))
output_file.write("----\n")
out = out.decode('utf-8').replace(usage_block.group(0), '')
else:
raise ValueError("No usage block matched in nodetool help output")
output_file.write(out)
except subprocess.CalledProcessError as cpe:
raise cpe
# for a given command, create the help file and an ADOC file to contain it
# for a given command, create the ADOC file to contain it
def create_adoc(command):
if command:
cmdName = command.group(0).strip()
cmdFilename = examplesdir + "/" + cmdName + ".txt"
adocFilename = outdir + "/" + cmdName + ".adoc"
with open(cmdFilename, "wb+") as cmdFile:
proc = Popen([nodetool, "help", cmdName], stdin=PIPE, stdout=PIPE)
(out, err) = proc.communicate()
cmdFile.write(out)
with open(adocFilename, "w+") as adocFile:
adocFile.write(commandADOCContent.format(cmdName,cmdName,cmdName))
@ -108,12 +77,5 @@ with open(outdir + "/nodetool.adoc", "w+") as output:
# create the command usage pages
with open(helpfilename, "r+") as helpfile:
for clis in batched(helpfile, batch_size):
threads = []
for commandLine in clis:
command = command_re.match(commandLine)
t = Thread(target=create_adoc, args=[command])
threads.append(t)
t.start()
for t in threads:
t.join()
for commandLine in helpfile:
create_adoc(command_re.match(commandLine))

113
test/unit/org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java
View File

@ -18,25 +18,26 @@
package org.apache.cassandra.tools.nodetool;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.tools.ToolRunner;
import org.apache.cassandra.tools.INodeProbeFactory;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool;
import org.apache.cassandra.tools.Output;
import static com.google.common.collect.Lists.newArrayList;
@ -45,47 +46,78 @@ import static com.google.common.collect.Lists.newArrayList;
* a separator between command hierarchy levels in the file names (e.g. {@code "info$threads"}) due to the fact that
* a command name can contain special characters like {@code -} or {@code _}.
* <p>
* The generator calls the {@code ./nodetool help} command to get the list of available commands and their descriptions,
* in order to generate the latest help output for each command be sure to run the generator after the jars are built
* (e.g. {@code ant jar}).
* The help output is produced in-process (single JVM); be sure to run the generator after the jars are built
* (e.g. {@code ant jar}). Pass {@code --dir <path>} to override the output directory and {@code --txt} to append
* a {@code .txt} extension (used by {@code doc/scripts/gen-nodetool-docs.py}).
*/
public class NodetoolHelpGenerator
{
private static final Logger logger = LoggerFactory.getLogger(NodetoolHelpGenerator.class);
private static final Map<String, String> ENV = ImmutableMap.of("JAVA_HOME", CassandraRelevantProperties.JAVA_HOME.getString());
private static final String NODETOOL_COMMAND_HELP_WRITE_DIR = "test/resources/nodetool/help/";
private static final String ROOT_COMMAND_FILE = "nodetool";
private static final String IGNORE_LINE = " With no arguments,";
private static final String NODETOOL_COMMAND_LIST_START_AFTER = "The most commonly used nodetool commands are:";
private static final String NODETOOL_SUBCOMMAND_LIST_START_AFTER = "COMMANDS";
private static final Pattern NODETOOL_COMMAND_DESCRIPTION_SPACES = Pattern.compile("^ {4}(\\S+)");
private static final Pattern NODETOOL_SUBCOMMAND_DESCRIPTION_SPACES = Pattern.compile("^ {8}(\\S+)");
private static final String COMMAND_FULL_NAME_SEPARATOR = "$";
private static final INodeProbeFactory NO_PROBE = new INodeProbeFactory()
{
public NodeProbe create(String host, int port) { throw new UnsupportedOperationException(); }
public NodeProbe create(String host, int port, String user, String pass) { throw new UnsupportedOperationException(); }
};
private final String writeDir;
private final String extension;
public NodetoolHelpGenerator(String writeDir, String extension)
{
this.writeDir = writeDir;
this.extension = extension;
}
/**
* Main method to generate help files for all nodetool commands to the {@code test/resources/nodetool/help/}.
* Main method to generate help files for all nodetool commands to the {@code test/resources/nodetool/help/}
* (or to the directory specified via {@code --dir}).
* <p>
* For example, the {@code nodetool help bootstrap resume} help output results in a file
* {@code test/resources/nodetool/help/bootstrap$resume}, where the {@code $} character
* is used as a separator for the subcommand. The arguments are passed as a list of commands
* to generate help files for. For example, {@code bootstrap resume} is passed.
* <p>
* By default, the files are written to {@code test/resources/nodetool/help/}.
* is used as a separator for the subcommand. Trailing positional arguments generate the help
* for a single command only (e.g. {@code bootstrap resume}).
*/
public static void main(String[] args)
{
List<String> commands = new ArrayList<>(List.of(args));
// commands.add("assassinate");
String dir = NODETOOL_COMMAND_HELP_WRITE_DIR;
String extension = "";
List<String> commands = new ArrayList<>();
for (int i = 0; i < args.length; i++)
{
switch (args[i])
{
case "--dir":
if (++i >= args.length)
throw new IllegalArgumentException("--dir requires a path");
dir = args[i];
break;
case "--txt":
extension = ".txt";
break;
default: commands.add(args[i]);
}
}
NodetoolHelpGenerator generator = new NodetoolHelpGenerator(dir, extension);
if (commands.isEmpty())
new NodetoolHelpGenerator().writeCommandsHelpOutput();
generator.writeCommandsHelpOutput();
else
new NodetoolHelpGenerator().writer(commands);
generator.writer(commands);
}
public void writeCommandsHelpOutput()
{
List<String> roots = find(() -> ToolRunner.invoke(ENV, newArrayList("bin/nodetool", "help")),
List<String> roots = find(() -> help(new ArrayList<>()),
NODETOOL_COMMAND_LIST_START_AFTER, NODETOOL_COMMAND_DESCRIPTION_SPACES);
writer(new ArrayList<>());
for (String command : roots)
writeToFileRecursively(newArrayList(command), this::writer);
@ -93,9 +125,7 @@ public class NodetoolHelpGenerator
private void writeToFileRecursively(List<String> hierarchy, Consumer<List<String>> writer)
{
List<String> subcommands = find(() -> ToolRunner.invoke(ENV, Lists.asList("bin/nodetool",
"help",
hierarchy.toArray(new String[0]))),
List<String> subcommands = find(() -> help(hierarchy),
NODETOOL_SUBCOMMAND_LIST_START_AFTER, NODETOOL_SUBCOMMAND_DESCRIPTION_SPACES);
for (String subcommand : subcommands)
{
@ -109,13 +139,12 @@ public class NodetoolHelpGenerator
public void writer(List<String> fullCommand)
{
ToolRunner.ToolResult result = ToolRunner.invoke(ENV, Lists.asList("bin/nodetool", "help",
fullCommand.toArray(new String[0])));
result.assertOnCleanExit();
String stdout = help(fullCommand);
String name = fullCommand.isEmpty() ? ROOT_COMMAND_FILE : String.join(COMMAND_FULL_NAME_SEPARATOR, fullCommand);
try
{
File commandHelpOut = new File(NODETOOL_COMMAND_HELP_WRITE_DIR, String.join(COMMAND_FULL_NAME_SEPARATOR, fullCommand)); //checkstyle: permit this instantiation
File commandHelpOut = new File(writeDir, name + extension); //checkstyle: permit this instantiation
boolean created = commandHelpOut.getParentFile().mkdirs();
if (created)
logger.debug("Created directory: {}", commandHelpOut.getParentFile().getAbsolutePath());
@ -126,7 +155,7 @@ public class NodetoolHelpGenerator
try (FileWriter fw = new FileWriter(commandHelpOut))
{
fw.write(result.getStdout().trim());
fw.write(stdout.trim());
fw.write("\n");
}
logger.info("The help is written for '{}' to '{}'", fullCommand, commandHelpOut.getAbsolutePath());
@ -137,11 +166,31 @@ public class NodetoolHelpGenerator
}
}
private static List<String> find(Supplier<ToolRunner.ToolResult> cmdResult, String afterLine, Pattern commandPattern)
private static String help(List<String> command)
{
ToolRunner.ToolResult result = cmdResult.get();
result.assertOnCleanExit();
String[] lines = result.getStdout().split("\n");
List<String> args = new ArrayList<>();
if (!command.isEmpty())
{
args.add("help");
args.addAll(command);
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayOutputStream err = new ByteArrayOutputStream();
PrintStream outStream = new PrintStream(out, true, StandardCharsets.UTF_8);
PrintStream errStream = new PrintStream(err, true, StandardCharsets.UTF_8);
int rc = new NodeTool(NO_PROBE, new Output(outStream, errStream)).execute(args.toArray(new String[0]));
outStream.flush();
errStream.flush();
String stderr = err.toString(StandardCharsets.UTF_8);
if (rc != 0 || !stderr.trim().isEmpty())
throw new RuntimeException("nodetool help " + String.join(" ", command) + " failed (rc=" + rc + "): "
+ stderr);
return out.toString(StandardCharsets.UTF_8);
}
private static List<String> find(Supplier<String> stdout, String afterLine, Pattern commandPattern)
{
String[] lines = stdout.get().split("\n");
List<String> commands = new ArrayList<>();
boolean start = false;
for (String line : lines)