From 03c86cfcb0dd31a9c904e2561e3f8a1f26357a44 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 25 Jan 2023 08:55:13 -0600 Subject: [PATCH] Add concurrency to adoc generation in gen-nodetool-docs.py Patch by brandonwiliams; reviewed by mck for CASSANDRA-18197 --- doc/scripts/gen-nodetool-docs.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/scripts/gen-nodetool-docs.py b/doc/scripts/gen-nodetool-docs.py index 1903dca876..efb79cf7a4 100644 --- a/doc/scripts/gen-nodetool-docs.py +++ b/doc/scripts/gen-nodetool-docs.py @@ -24,6 +24,10 @@ import sys import subprocess from subprocess import PIPE from subprocess import Popen +from itertools import islice +from threading import Thread + +batch_size = 3 if(os.environ.get("SKIP_NODETOOL") == "1"): sys.exit(0) @@ -36,6 +40,16 @@ helpfilename = outdir + "/nodetool.txt" command_re = re.compile("( )([_a-z]+)") commandADOCContent = "== {0}\n\n== Usage\n[source,plaintext]\n----\ninclude::example$TEXT/NODETOOL/{0}.txt[]\n----\n" +# https://docs.python.org/3/library/itertools.html#itertools-recipes +def batched(iterable, n): + "Batch data into tuples of length n. The last batch may be shorter." + # batched('ABCDEFG', 3) --> ABC DEF G + if n < 1: + raise ValueError('n must be at least one') + it = iter(iterable) + while (batch := tuple(islice(it, n))): + yield batch + # create the documentation directory if not os.path.exists(outdir): os.makedirs(outdir) @@ -78,6 +92,12 @@ with open(outdir + "/nodetool.adoc", "w+") as output: # create the command usage pages with open(helpfilename, "r+") as helpfile: - for commandLine in helpfile: - command = command_re.match(commandLine) - create_adoc(command) + for clis in batched(helpfile, batch_size): + threads = [] + for commandLine in clis: + command = command_re.match(commandLine) + t = Thread(target=create_adoc, args=[command]) + threads.append(t) + t.start() + for t in threads: + t.join()