natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove golang dependency in gen-doc and replace with python implementation 

patch by Maxim Muzafarov; reviewed by Dmitry Konstantinov for CASSANDRA-21432
This commit is contained in:
Maxim Muzafarov 2026-06-08 20:40:00 +02:00
parent 1c19e860d7
commit 8c992cb6a5
No known key found for this signature in database
GPG Key ID: 7FEC714D84388C16
7 changed files with 380 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.build/docker/bullseye-build.docker
View File

@ -53,25 +53,6 @@ RUN pip install --upgrade pip
# dependencies for .build/ci/ci_parser.py
RUN pip install beautifulsoup4==4.12.3 jinja2==3.1.3
# install golang. GO_VERSION_SHA must be updated with VERSION
RUN sh -c '\
GO_VERSION="1.24.3" ;\
GO_VERSION_SHAS="3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8 a463cb59382bd7ae7d8f4c68846e73c4d589f223c589ac76871b66811ded7836 13e6fe3fcf65689d77d40e633de1e31c6febbdbcb846eb05fc2434ed2213e92b 64a3fa22142f627e78fac3018ce3d4aeace68b743eff0afda8aae0411df5e4fb" ;\
GO_OS=linux ;\
[ $(uname) = "Darwin" ] && GO_OS=darwin ;\
GO_PLATFORM=amd64 ;\
[ $(uname -m) = "aarch64" ] && GO_PLATFORM=arm64 ;\
GO_TAR="go${GO_VERSION}.${GO_OS}-${GO_PLATFORM}.tar.gz" ;\
curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 30 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\
echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\
tar -C /usr/local -xzf $GO_TAR ;\
rm $GO_TAR'
ENV GOROOT="/usr/local/go"
ENV GOPATH="$BUILD_HOME/go"
ENV PATH="$PATH:/usr/local/go/bin"
# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.0.9
* Remove golang dependency in gen-doc and replace with python implementation (CASSANDRA-21432)
* Use estimated compressed size for tables to check if there is enough free space for a compaction (CASSANDRA-21245)
* Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348)
* Ensure SAI sends range tombstones to the coordinator for queries on static columns (CASSANDRA-21332)

4
build.xml
View File

@ -980,7 +980,7 @@
<include name="tools/bin/*"/>
<exclude name="tools/bin/*.in.sh" />
<include name=".build/**/*.sh"/>
<include name="doc/scripts/process-native-protocol-specs-in-docker.sh"/>
<include name="doc/scripts/gen-native-protocol-docs.sh"/>
</tarfileset>
</tar>
@ -1330,7 +1330,7 @@
<fileset file="${test.conf}/cassandra.yaml"/>
<fileset file="${test.conf}/storage_compatibility_mode_none.yaml"/>
</concat>
<testmacrohelper inputdir="${test.dir}/${test.classlistprefix}" filelist="@{test.file.list}"
<testmacrohelper inputdir="${test.dir}/${test.classlistprefix}" filelist="@{test.file.list}"
exclude="**/*.java" timeout="${test.timeout}" testtag="oa">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>

4
doc/Makefile
View File

@ -14,7 +14,7 @@
GENERATE_ANTORA_YML = ./scripts/gen-antora-yml.py
GENERATE_NODETOOL_DOCS = ./scripts/gen-nodetool-docs.py
MAKE_CASSANDRA_YAML = ./scripts/convert_yaml_to_adoc.py ../conf/cassandra.yaml ./modules/cassandra/pages/managing/configuration/cass_yaml_file.adoc
PROCESS_NATIVE_PROC_SPECS = ./scripts/process-native-protocol-specs-in-docker.sh
GEN_NATIVE_PROTOCOL_DOCS = ./scripts/gen-native-protocol-docs.sh
.PHONY: html
html:
@ -27,4 +27,4 @@ gen-asciidoc:
@mkdir -p modules/cassandra/examples/TEXT/NODETOOL
python3 $(GENERATE_NODETOOL_DOCS)
python3 $(MAKE_CASSANDRA_YAML)
$(PROCESS_NATIVE_PROC_SPECS)
$(GEN_NATIVE_PROTOCOL_DOCS)

347
doc/scripts/cqlprotodoc.py Executable file
View File

@ -0,0 +1,347 @@
#!/usr/bin/env python3
"""Generate native-protocol HTML and asciidoc summary from .spec files."""
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import sys
import re
import html
import io
from pathlib import Path
from typing import List
_comment_re = re.compile(r'^#\s?(.*)$')
_empty_re = re.compile(r'^\s*$')
_title_re = re.compile(r'^\s+(.*)\s*$')
_heading_re = re.compile(r'^(?P<indent>\s*)(?P<number>\d+(?:\.\d+)*)\.?\s+(?P<title>[A-Za-z_].+)$')
_toc_entry_re = re.compile(r'^(?P<number>\d+(?:\.\d+)*)\.?\s+(?P<title>.+)$')
_url_re = re.compile(r'(https?://[^\s)]+)')
_URL_TRAILING_PUNCT = '.,;:!?'
_protocol_filename_re = re.compile(r'^native_protocol_v(\d+)\.(?:spec|html)$')
def _skip_blank(lines: List[str], idx: int) -> int:
while idx < len(lines) and _empty_re.match(lines[idx]):
idx += 1
return idx
def parse_spec_file(path: Path) -> dict:
"""Parse a native_protocol_v*.spec file into license, title, TOC, and sections."""
text = path.read_text(encoding='utf-8')
lines = text.splitlines()
idx = 0
# License
license_lines = []
while idx < len(lines):
m = _comment_re.match(lines[idx])
if not m:
break
license_lines.append(m.group(1))
idx += 1
idx = _skip_blank(lines, idx)
# Titles
m = _title_re.match(lines[idx]) if idx < len(lines) else None
if not m:
sys.exit(f"Parse error: missing or malformed title at line {idx + 1}")
title = m.group(1)
idx += 1
idx = _skip_blank(lines, idx)
# Table of Contents
if idx >= len(lines) or lines[idx] != "Table of Contents":
sys.exit(f"Parse error: expected 'Table of Contents' at line {idx + 1}")
idx += 1
idx = _skip_blank(lines, idx)
# TOC
toc = []
while idx < len(lines) and lines[idx].strip():
line = lines[idx].strip()
m = _toc_entry_re.match(line)
if not m:
sys.exit(f"Parse error: bad TOC entry at line {idx + 1}")
toc.append({'number': m.group('number'), 'title': m.group('title')})
idx += 1
idx = _skip_blank(lines, idx)
# Sections distinguishing real headings from prose/list-items
sections = []
current = None
for line in lines[idx:]:
m = _heading_re.match(line)
if m:
num = m.group('number')
sec_title = m.group('title').rstrip()
if '.' in num or m.group('indent') == '':
if current:
sections.append(current)
current = {'number': num, 'title': sec_title, 'body': []}
continue
if current:
current['body'].append(line)
if current:
sections.append(current)
return {
'license': license_lines,
'title': title,
'toc': toc,
'sections': sections,
}
def build_toc_tree(entries, nums):
"""Build a nested TOC tree from flat entries; mark which numbers exist as sections."""
root = {'children': []}
stack = [root]
for e in entries:
lvl = e['number'].count('.') + 1
stack = stack[:lvl]
parent = stack[-1]
node = {'entry': e, 'exists': e['number'] in nums, 'children': []}
parent['children'].append(node)
stack.append(node)
return root['children']
_section_multi_re = re.compile(
r'([sS]ections)(\s+\d+(?:\.\d+)*(?:,?\s+(?:and\s+)?\d+(?:\.\d+)*)+)'
)
_section_single_re = re.compile(r'([sS]ection (\d+(?:\.\d+)*))')
_num_re = re.compile(r'\d+(?:\.\d+)*')
def _linkify_section_refs(text: str) -> str:
def repl_multi(m):
return m.group(1) + _num_re.sub(
lambda n: f'<a href="#s{n.group(0)}">{n.group(0)}</a>', m.group(2))
text = _section_multi_re.sub(repl_multi, text)
text = _section_single_re.sub(
lambda m: f'<a href="#s{m.group(2)}">{m.group(1)}</a>', text)
return text
def _linkify_url(m):
url = m.group(1)
trailing = ''
while url and url[-1] in _URL_TRAILING_PUNCT:
trailing = url[-1] + trailing
url = url[:-1]
return f'<a href="{url}">{url}</a>{trailing}'
def format_body(lines):
"""Render section body lines as escaped HTML with URL and section-ref linkification."""
text = "\n".join(lines)
if text.startswith('\n'):
text = text[1:]
escaped = html.escape(text)
with_urls = _url_re.sub(_linkify_url, escaped)
linked = _linkify_section_refs(with_urls)
# Transcode entity names to byte-match the previous (go) tools output.
linked = linked.replace('&quot;', '&#34;').replace('&#x27;', '&#39;')
return '<pre>' + linked + '</pre>'
def build_sections(secs):
"""Convert raw parsed sections into render-ready dicts with HTML body and heading level."""
return [{
'number': s['number'],
'title': s['title'],
'level': s['number'].count('.') + 2,
'body_html': format_body(s['body'])
} for s in secs]
def _render_toc(nodes, out, indent):
pad = ' ' * indent
for node in nodes:
num = node['entry']['number']
node_title = html.escape(node['entry']['title'])
out.write(f'{pad}<li id="toc{num}">\n')
out.write(f'{pad} {num}\n')
if node['exists']:
out.write(f'{pad} <a href="#s{num}">{node_title}</a>\n')
else:
out.write(f'{pad} {node_title}\n')
if node['children']:
out.write(f'{pad} <ol>\n')
_render_toc(node['children'], out, indent + 2)
out.write(f'{pad} </ol>\n')
out.write(f'{pad}</li>\n')
def render_html(title, license_lines, toc_tree, sections):
"""Render the full HTML document for a single protocol version."""
out = io.StringIO()
t_esc = html.escape(title)
out.write('<!DOCTYPE html>\n')
out.write('<html>\n')
out.write('<head>\n')
out.write(' <meta charset="utf-8">\n')
out.write(f' <title>{t_esc}</title>\n')
out.write(' <style>\n')
out.write(' nav ol { margin: 0; padding: 0; padding-left: 1em; }\n')
out.write(' nav li { list-style: none; }\n')
out.write(' nav.top ul { margin: 0; padding: 0; background: #eee; color: black; }\n')
out.write(' nav.top ul li { display: inline-block; }\n')
out.write(' </style>\n')
out.write('</head>\n')
out.write('<body>\n')
for line in license_lines:
out.write(f' <!-- {html.escape(line)} -->\n')
out.write(f' <h1>{t_esc}</h1>\n')
out.write(' <h2>Table of Contents</h2>\n')
out.write(' <nav>\n')
out.write(' <ol>\n')
_render_toc(toc_tree, out, 3)
out.write(' </ol>\n')
out.write(' </nav>\n')
for sec in sections:
lvl = sec['level']
num = sec['number']
sec_title = html.escape(sec['title'])
out.write(f' <h{lvl} id="s{num}">{num} {sec_title}</h{lvl}>\n')
out.write(f' {sec["body_html"]}\n')
out.write('</body>\n')
out.write('</html>\n')
return out.getvalue()
def main(): # pylint: disable=too-many-locals
"""CLI entrypoint: render one HTML page per spec file plus an asciidoc summary."""
parser = argparse.ArgumentParser(
description="Generate native-protocol HTML and asciidoc summary from .spec files."
)
parser.add_argument(
'--spec-dir', type=Path, default=Path('.'),
help="Directory containing native_protocol_v*.spec files (default: cwd)."
)
parser.add_argument(
'--attach-dir', type=Path, default=Path('modules/cassandra/attachments'),
help="Output directory for per-version HTML files."
)
parser.add_argument(
'--summary-adoc', type=Path,
default=Path('modules/cassandra/pages/reference/native-protocol.adoc'),
help="Output path for the generated asciidoc summary."
)
args = parser.parse_args()
spec_dir = args.spec_dir
attach_dir = args.attach_dir
summary_adoc = args.summary_adoc
if not spec_dir.is_dir():
sys.exit(f"Spec directory does not exist: {spec_dir.resolve()}")
attach_dir.mkdir(parents=True, exist_ok=True)
summary_adoc.parent.mkdir(parents=True, exist_ok=True)
specs = sorted(
(p for p in spec_dir.glob('native_protocol_v*.spec')
if _protocol_filename_re.match(p.name)),
key=lambda p: int(_protocol_filename_re.match(p.name).group(1)),
reverse=True,
)
if not specs:
sys.exit(f"No native_protocol_v*.spec files found in {spec_dir.resolve()}")
for sp in specs:
version = _protocol_filename_re.match(sp.name).group(1)
hp = attach_dir / f'native_protocol_v{version}.html'
doc = parse_spec_file(sp)
toc_tree = build_toc_tree(doc['toc'], {s['number'] for s in doc['sections']})
sections = build_sections(doc['sections'])
rendered = render_html(doc['title'], doc['license'], toc_tree, sections)
hp.write_text(rendered, encoding='utf-8')
print(f"-> {hp}")
nav_js = """[source, js]
++++
<script>
function setNavigation() {
var containers = document.querySelectorAll('.sect1');
containers.forEach(function (container) {
var preElements = container.querySelectorAll('pre');
preElements.forEach(function(preElement) {
if (!preElement.textContent.trim()) {
preElement.remove();
}
});
var h1Elements = container.querySelectorAll('h1');
h1Elements.forEach(function(h1Element) {
h1Element.remove();
});
var navLinks = container.querySelectorAll('nav a, pre a');
navLinks.forEach(function (link) {
link.addEventListener('click', function (event) {
event.preventDefault();
var section = link.getAttribute('href').replace("#", '');
var targetSection = container.querySelector('h2[id="' + section + '"]')
|| container.querySelector('h3[id="' + section + '"]')
|| container.querySelector('h4[id="' + section + '"]')
|| container.querySelector('h5[id="' + section + '"]');
if (targetSection) {
targetSection.scrollIntoView({ behavior: 'smooth' });
}
});
});
});
}
window.onload = function() {
setNavigation()
}
</script>
++++
"""
html_files = sorted(
(p for p in attach_dir.glob('native_protocol_v*.html')
if _protocol_filename_re.match(p.name)),
key=lambda p: int(_protocol_filename_re.match(p.name).group(1)),
reverse=True,
)
with summary_adoc.open('w', encoding='utf-8') as f:
f.write("= Native Protocol Versions\n")
f.write(":page-layout: default\n\n")
for file in html_files:
ver = _protocol_filename_re.match(file.name).group(1)
f.write(f"== Native Protocol Version {ver}\n\n")
f.write("[source, html]\n++++\n")
f.write(f"include::cassandra:attachment${file.name}[Version {ver}]\n")
f.write("++++\n\n")
f.write(nav_js)
print(f"-> {summary_adoc}")
if __name__ == '__main__':
main()

28
doc/scripts/gen-native-protocol-docs.sh Executable file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -euo pipefail
[ "x${SCRIPT_DIR:-}" != "x" ] || SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
[ "x${PYTOOL:-}" != "x" ] || PYTOOL="$SCRIPT_DIR/cqlprotodoc.py"
echo "Processing native protocol specs..."
python3 "$PYTOOL"
echo "Done"
exit 0

146
doc/scripts/process-native-protocol-specs-in-docker.sh
View File

@ -1,146 +0,0 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[ -f "../build.xml" ] || { echo "build.xml must exist (current directory needs to be doc/ in cassandra repo"; exit 1; }
[ -f "antora.yml" ] || { echo "antora.yml must exist (current directory needs to be doc/ in cassandra repo"; exit 1; }
# Variables
GO_VERSION="1.23.1"
TMPDIR="${TMPDIR:-/tmp}"
check_go_version() {
if command -v go &>/dev/null; then
local installed_version=$(go version | awk '{print $3}' | sed 's/go//')
if [ "$(printf '%s\n' "$GO_VERSION" "$installed_version" | sort -V | head -n1)" = "$GO_VERSION" ]; then
echo "Detected Go $installed_version (>= $GO_VERSION)"
return 0
else
echo "Detected unsupported Go $installed_version (< $GO_VERSION), please update to supported version."
fi
else
echo "No Go installation detected, please install Go (>= $GO_VERSION)"
fi
return 1
}
if ! check_go_version; then
echo " Please install/upgrade Golang for 'ant gen-doc', or specify '-Dant.gen-doc.skip=true' to skip this step."
echo " For download and installation instructions see https://go.dev/doc/install"
exit 1
fi
# Step 1: Building the parser
echo "Building the cqlprotodoc..."
DIR="$(pwd)"
cd "${TMPDIR}"
rm -rf "${TMPDIR}/cassandra-website"
git clone -n --depth=1 --filter=tree:0 https://github.com/apache/cassandra-website
if [ $? != "0" ]; then
echo "Error occured while cloning https://github.com/apache/cassandra-website"
exit 1
fi
cd "${TMPDIR}/cassandra-website"
git sparse-checkout set --no-cone /cqlprotodoc
git checkout
cd "${TMPDIR}/cassandra-website/cqlprotodoc"
rm -rf "${TMPDIR}/cqlprotodoc"
go build -o "$TMPDIR"/cqlprotodoc
# Step 2: Process the spec files using the parser
echo "Processing the .spec files..."
cd "${DIR}"
output_dir="modules/cassandra/attachments"
mkdir -p "${output_dir}"
"$TMPDIR"/cqlprotodoc . "${output_dir}"
if ! ls ${output_dir}/native_protocol_v*.html > /dev/null 2>&1; then
echo "failed: No native_protocol_v*.html files generated in ${output_dir}"
exit 1
fi
# Step 4: Generate summary file
summary_file="modules/cassandra/pages/reference/native-protocol.adoc"
# Write the header
echo "= Native Protocol Versions" > "$summary_file"
echo ":page-layout: default" >> "$summary_file"
echo >> "$summary_file"
# Loop through the files from step 2 in reverse version order
for file in $(ls ${output_dir}/native_protocol_v*.html | sort -r | awk -F/ '{print $NF}'); do
version=$(echo "$file" | sed -E 's/native_protocol_v([0-9]+)\.html/\1/')
echo "== Native Protocol Version $version" >> "$summary_file"
echo >> "$summary_file"
echo "[source, html]" >> "$summary_file"
echo "++++" >> "$summary_file"
echo "include::cassandra:attachment\$$file[Version $version]" >> "$summary_file"
echo "++++" >> "$summary_file"
echo >> "$summary_file"
done
# Navigation setup
echo "[source, js]" >> "$summary_file"
echo "++++" >> "$summary_file"
echo "<script>" >> "$summary_file"
echo " function setNavigation() {" >> "$summary_file"
echo " var containers = document.querySelectorAll('.sect1');" >> "$summary_file"
echo >> "$summary_file"
echo " containers.forEach(function (container) {" >> "$summary_file"
echo " var preElements = container.querySelectorAll('pre');" >> "$summary_file"
echo " preElements.forEach(function(preElement) {" >> "$summary_file"
echo " if (!preElement.textContent.trim()) {" >> "$summary_file"
echo " preElement.remove();" >> "$summary_file"
echo " }" >> "$summary_file"
echo " });" >> "$summary_file"
echo " var h1Elements = container.querySelectorAll('h1');" >> "$summary_file"
echo " h1Elements.forEach(function(h1Element) {" >> "$summary_file"
echo " h1Element.remove();" >> "$summary_file"
echo " });" >> "$summary_file"
echo >> "$summary_file"
echo " var navLinks = container.querySelectorAll('nav a, pre a');" >> "$summary_file"
echo >> "$summary_file"
echo " navLinks.forEach(function (link) {" >> "$summary_file"
echo " link.addEventListener('click', function (event) {" >> "$summary_file"
echo >> "$summary_file"
echo " event.preventDefault();" >> "$summary_file"
echo " var section = link.getAttribute('href').replace(\"#\", '');" >> "$summary_file"
echo >> "$summary_file"
echo " var targetSection = container.querySelector('h2[id=\"' + section + '\"]') || container.querySelector('h3[id=\"' + section + '\"]') || container.querySelector('h4[id=\"' + section + '\"]') || container.querySelector('h5[id=\"' + section + '\"]');" >> "$summary_file"
echo >> "$summary_file"
echo " if (targetSection) {" >> "$summary_file"
echo " targetSection.scrollIntoView({ behavior: 'smooth' });" >> "$summary_file"
echo " }" >> "$summary_file"
echo " });" >> "$summary_file"
echo " });" >> "$summary_file"
echo " });" >> "$summary_file"
echo " }" >> "$summary_file"
echo >> "$summary_file"
echo " window.onload = function() {" >> "$summary_file"
echo " setNavigation()" >> "$summary_file"
echo " }" >> "$summary_file"
echo " </script>" >> "$summary_file"
# Step 3: Cleanup - Remove the Cassandra and parser directories
echo "Cleaning up..."
cd "${DIR}"
rm -rf "${TMPDIR}/cassandra-website" "${TMPDIR}/cqlprotodoc" 2>/dev/null
echo "Script completed successfully."