diff --git a/.build/docker/almalinux-build.docker b/.build/docker/almalinux-build.docker index ed7cc36338..ad8d904552 100644 --- a/.build/docker/almalinux-build.docker +++ b/.build/docker/almalinux-build.docker @@ -81,27 +81,6 @@ RUN sh -c '\ # Install Antora for documentation generation RUN npm i -g @antora/cli@2.3 @antora/site-generator-default@2.3 @djencks/asciidoctor-openblock -# install golang. GO_VERSION_SHA must be updated with VERSION - -RUN sh -c '\ - set -ex ;\ - GO_VERSION="1.26.1" ;\ - GO_VERSION_SHAS="031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7 65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08 353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f" ;\ - GO_OS=linux ;\ - [ $(uname) = "Darwin" ] && GO_OS=darwin ;\ - GO_PLATFORM=amd64 ;\ - [ $(uname -m) = "aarch64" ] && GO_PLATFORM=arm64 ;\ - GO_TAR="go${GO_VERSION}.${GO_OS}-${GO_PLATFORM}.tar.gz" ;\ - curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 3600 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\ - GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\ - echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\ - tar -C /usr/local -xzf $GO_TAR ;\ - rm $GO_TAR' - -ENV GOROOT="/usr/local/go" -ENV GOPATH="$BUILD_HOME/go" -ENV PATH="$PATH:/usr/local/go/bin" - # Prepopulate Maven repository with dependencies from all branches. see _create_user.sh COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh RUN alternatives --set java $(alternatives --display java | grep "family java-11-openjdk" | cut -d' ' -f1) diff --git a/.build/docker/debian-build.docker b/.build/docker/debian-build.docker index 18eacc4b8e..3ef612eea7 100644 --- a/.build/docker/debian-build.docker +++ b/.build/docker/debian-build.docker @@ -87,23 +87,6 @@ RUN sh -c '\ # Install Antora for documentation generation RUN npm i -g @antora/cli@2.3 @antora/site-generator-default@2.3 @djencks/asciidoctor-openblock -# install golang. GO_VERSION_SHA must be updated with VERSION -RUN sh -c '\ - set -ex ;\ - GO_VERSION="1.26.1" ;\ - GO_VERSION_SHAS="031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7 65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08 353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f" ;\ - GO_OS=linux ;\ - GO_TAR="go${GO_VERSION}.${GO_OS}-$(dpkg --print-architecture).tar.gz" ;\ - curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 3600 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\ - GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\ - echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\ - tar -C /usr/local -xzf $GO_TAR ;\ - rm $GO_TAR' - -ENV GOROOT="/usr/local/go" -ENV GOPATH="$BUILD_HOME/go" -ENV PATH="$PATH:/usr/local/go/bin" - # allow lower UIDs and GIDs RUN sed -i 's/UID_MIN 1000/UID_MIN 100/' /etc/login.defs RUN sed -i 's/UID_MIN 1000/UID_MIN 10/' /etc/login.defs diff --git a/CHANGES.txt b/CHANGES.txt index 294669d9ad..8e4c39b56f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -40,6 +40,7 @@ * Fix a removed TTLed row re-appearance in a materialized view after a cursor compaction (CASSANDRA-21152) * Rework ZSTD dictionary compression logic to create a trainer per training (CASSANDRA-21209) Merged from 5.0: + * Remove golang dependency in gen-doc and replace with python implementation (CASSANDRA-21432) * Use estimated compressed size for tables to check if there is enough free space for a compaction (CASSANDRA-21245) * Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348) * Ensure SAI sends range tombstones to the coordinator for queries on static columns (CASSANDRA-21332) diff --git a/build.xml b/build.xml index 34cb09ce26..47cd619631 100644 --- a/build.xml +++ b/build.xml @@ -1135,7 +1135,7 @@ - + diff --git a/doc/Makefile b/doc/Makefile index 3b7225f444..940d0d18b5 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -14,7 +14,7 @@ GENERATE_ANTORA_YML = ./scripts/gen-antora-yml.py GENERATE_NODETOOL_DOCS = ./scripts/gen-nodetool-docs.py MAKE_CASSANDRA_YAML = ./scripts/convert_yaml_to_adoc.py ../conf/cassandra.yaml ./modules/cassandra/pages/managing/configuration/cass_yaml_file.adoc -PROCESS_NATIVE_PROC_SPECS = ./scripts/process-native-protocol-specs-in-docker.sh +GEN_NATIVE_PROTOCOL_DOCS = ./scripts/gen-native-protocol-docs.sh TABS_BLOCK_JS_URL = https://raw.githubusercontent.com/apache/cassandra-website/trunk/site-content/lib/tabs-block.js .PHONY: init @@ -49,7 +49,7 @@ gen-asciidoc: @mkdir -p modules/cassandra/examples/TEXT/NODETOOL python3 $(GENERATE_NODETOOL_DOCS) python3 $(MAKE_CASSANDRA_YAML) - $(PROCESS_NATIVE_PROC_SPECS) + $(GEN_NATIVE_PROTOCOL_DOCS) .PHONY: manpages manpages: diff --git a/doc/scripts/cqlprotodoc.py b/doc/scripts/cqlprotodoc.py new file mode 100755 index 0000000000..8abedd997c --- /dev/null +++ b/doc/scripts/cqlprotodoc.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""Generate native-protocol HTML and asciidoc summary from .spec files.""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys +import re +import html +import io +from pathlib import Path +from typing import List + +_comment_re = re.compile(r'^#\s?(.*)$') +_empty_re = re.compile(r'^\s*$') +_title_re = re.compile(r'^\s+(.*)\s*$') +_heading_re = re.compile(r'^(?P\s*)(?P\d+(?:\.\d+)*)\.?\s+(?P[A-Za-z_].+)$') +_toc_entry_re = re.compile(r'^(?P<number>\d+(?:\.\d+)*)\.?\s+(?P<title>.+)$') +_url_re = re.compile(r'(https?://[^\s)]+)') +_URL_TRAILING_PUNCT = '.,;:!?' +_protocol_filename_re = re.compile(r'^native_protocol_v(\d+)\.(?:spec|html)$') + + +def _skip_blank(lines: List[str], idx: int) -> int: + while idx < len(lines) and _empty_re.match(lines[idx]): + idx += 1 + return idx + + +def parse_spec_file(path: Path) -> dict: + """Parse a native_protocol_v*.spec file into license, title, TOC, and sections.""" + text = path.read_text(encoding='utf-8') + lines = text.splitlines() + idx = 0 + + # License + license_lines = [] + while idx < len(lines): + m = _comment_re.match(lines[idx]) + if not m: + break + license_lines.append(m.group(1)) + idx += 1 + idx = _skip_blank(lines, idx) + + # Titles + m = _title_re.match(lines[idx]) if idx < len(lines) else None + if not m: + sys.exit(f"Parse error: missing or malformed title at line {idx + 1}") + title = m.group(1) + idx += 1 + idx = _skip_blank(lines, idx) + + # Table of Contents + if idx >= len(lines) or lines[idx] != "Table of Contents": + sys.exit(f"Parse error: expected 'Table of Contents' at line {idx + 1}") + idx += 1 + idx = _skip_blank(lines, idx) + + # TOC + toc = [] + while idx < len(lines) and lines[idx].strip(): + line = lines[idx].strip() + m = _toc_entry_re.match(line) + if not m: + sys.exit(f"Parse error: bad TOC entry at line {idx + 1}") + toc.append({'number': m.group('number'), 'title': m.group('title')}) + idx += 1 + idx = _skip_blank(lines, idx) + + # Sections distinguishing real headings from prose/list-items + sections = [] + current = None + for line in lines[idx:]: + m = _heading_re.match(line) + if m: + num = m.group('number') + sec_title = m.group('title').rstrip() + if '.' in num or m.group('indent') == '': + if current: + sections.append(current) + current = {'number': num, 'title': sec_title, 'body': []} + continue + if current: + current['body'].append(line) + + if current: + sections.append(current) + + return { + 'license': license_lines, + 'title': title, + 'toc': toc, + 'sections': sections, + } + + +def build_toc_tree(entries, nums): + """Build a nested TOC tree from flat entries; mark which numbers exist as sections.""" + root = {'children': []} + stack = [root] + for e in entries: + lvl = e['number'].count('.') + 1 + stack = stack[:lvl] + parent = stack[-1] + node = {'entry': e, 'exists': e['number'] in nums, 'children': []} + parent['children'].append(node) + stack.append(node) + return root['children'] + + +_section_multi_re = re.compile( + r'([sS]ections)(\s+\d+(?:\.\d+)*(?:,?\s+(?:and\s+)?\d+(?:\.\d+)*)+)' +) + + +_section_single_re = re.compile(r'([sS]ection (\d+(?:\.\d+)*))') +_num_re = re.compile(r'\d+(?:\.\d+)*') + + +def _linkify_section_refs(text: str) -> str: + def repl_multi(m): + return m.group(1) + _num_re.sub( + lambda n: f'<a href="#s{n.group(0)}">{n.group(0)}</a>', m.group(2)) + text = _section_multi_re.sub(repl_multi, text) + text = _section_single_re.sub( + lambda m: f'<a href="#s{m.group(2)}">{m.group(1)}</a>', text) + return text + + +def _linkify_url(m): + url = m.group(1) + trailing = '' + while url and url[-1] in _URL_TRAILING_PUNCT: + trailing = url[-1] + trailing + url = url[:-1] + return f'<a href="{url}">{url}</a>{trailing}' + + +def format_body(lines): + """Render section body lines as escaped HTML with URL and section-ref linkification.""" + text = "\n".join(lines) + if text.startswith('\n'): + text = text[1:] + escaped = html.escape(text) + with_urls = _url_re.sub(_linkify_url, escaped) + linked = _linkify_section_refs(with_urls) + # Transcode entity names to byte-match the previous (go) tools output. + linked = linked.replace('"', '"').replace(''', ''') + return '<pre>' + linked + '</pre>' + + +def build_sections(secs): + """Convert raw parsed sections into render-ready dicts with HTML body and heading level.""" + return [{ + 'number': s['number'], + 'title': s['title'], + 'level': s['number'].count('.') + 2, + 'body_html': format_body(s['body']) + } for s in secs] + + +def _render_toc(nodes, out, indent): + pad = ' ' * indent + for node in nodes: + num = node['entry']['number'] + node_title = html.escape(node['entry']['title']) + out.write(f'{pad}<li id="toc{num}">\n') + out.write(f'{pad} {num}\n') + if node['exists']: + out.write(f'{pad} <a href="#s{num}">{node_title}</a>\n') + else: + out.write(f'{pad} {node_title}\n') + if node['children']: + out.write(f'{pad} <ol>\n') + _render_toc(node['children'], out, indent + 2) + out.write(f'{pad} </ol>\n') + out.write(f'{pad}</li>\n') + + +def render_html(title, license_lines, toc_tree, sections): + """Render the full HTML document for a single protocol version.""" + out = io.StringIO() + t_esc = html.escape(title) + out.write('<!DOCTYPE html>\n') + out.write('<html>\n') + out.write('<head>\n') + out.write(' <meta charset="utf-8">\n') + out.write(f' <title>{t_esc}\n') + out.write(' \n') + out.write('\n') + out.write('\n') + for line in license_lines: + out.write(f' \n') + out.write(f'

{t_esc}

\n') + out.write('

Table of Contents

\n') + out.write(' \n') + for sec in sections: + lvl = sec['level'] + num = sec['number'] + sec_title = html.escape(sec['title']) + out.write(f' {num} {sec_title}\n') + out.write(f' {sec["body_html"]}\n') + out.write('\n') + out.write('\n') + return out.getvalue() + + +def main(): # pylint: disable=too-many-locals + """CLI entrypoint: render one HTML page per spec file plus an asciidoc summary.""" + parser = argparse.ArgumentParser( + description="Generate native-protocol HTML and asciidoc summary from .spec files." + ) + parser.add_argument( + '--spec-dir', type=Path, default=Path('.'), + help="Directory containing native_protocol_v*.spec files (default: cwd)." + ) + parser.add_argument( + '--attach-dir', type=Path, default=Path('modules/cassandra/attachments'), + help="Output directory for per-version HTML files." + ) + parser.add_argument( + '--summary-adoc', type=Path, + default=Path('modules/cassandra/pages/reference/native-protocol.adoc'), + help="Output path for the generated asciidoc summary." + ) + args = parser.parse_args() + + spec_dir = args.spec_dir + attach_dir = args.attach_dir + summary_adoc = args.summary_adoc + + if not spec_dir.is_dir(): + sys.exit(f"Spec directory does not exist: {spec_dir.resolve()}") + + attach_dir.mkdir(parents=True, exist_ok=True) + summary_adoc.parent.mkdir(parents=True, exist_ok=True) + + specs = sorted( + (p for p in spec_dir.glob('native_protocol_v*.spec') + if _protocol_filename_re.match(p.name)), + key=lambda p: int(_protocol_filename_re.match(p.name).group(1)), + reverse=True, + ) + if not specs: + sys.exit(f"No native_protocol_v*.spec files found in {spec_dir.resolve()}") + + for sp in specs: + version = _protocol_filename_re.match(sp.name).group(1) + hp = attach_dir / f'native_protocol_v{version}.html' + doc = parse_spec_file(sp) + toc_tree = build_toc_tree(doc['toc'], {s['number'] for s in doc['sections']}) + sections = build_sections(doc['sections']) + rendered = render_html(doc['title'], doc['license'], toc_tree, sections) + hp.write_text(rendered, encoding='utf-8') + print(f"-> {hp}") + + nav_js = """[source, js] +++++ + +++++ +""" + + html_files = sorted( + (p for p in attach_dir.glob('native_protocol_v*.html') + if _protocol_filename_re.match(p.name)), + key=lambda p: int(_protocol_filename_re.match(p.name).group(1)), + reverse=True, + ) + with summary_adoc.open('w', encoding='utf-8') as f: + f.write("= Native Protocol Versions\n") + f.write(":page-layout: default\n\n") + for file in html_files: + ver = _protocol_filename_re.match(file.name).group(1) + f.write(f"== Native Protocol Version {ver}\n\n") + f.write("[source, html]\n++++\n") + f.write(f"include::cassandra:attachment${file.name}[Version {ver}]\n") + f.write("++++\n\n") + f.write(nav_js) + print(f"-> {summary_adoc}") + + +if __name__ == '__main__': + main() diff --git a/doc/scripts/gen-native-protocol-docs.sh b/doc/scripts/gen-native-protocol-docs.sh new file mode 100755 index 0000000000..17bef66e74 --- /dev/null +++ b/doc/scripts/gen-native-protocol-docs.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +[ "x${SCRIPT_DIR:-}" != "x" ] || SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +[ "x${PYTOOL:-}" != "x" ] || PYTOOL="$SCRIPT_DIR/cqlprotodoc.py" + +echo "Processing native protocol specs..." +python3 "$PYTOOL" + +echo "Done" +exit 0 diff --git a/doc/scripts/process-native-protocol-specs-in-docker.sh b/doc/scripts/process-native-protocol-specs-in-docker.sh deleted file mode 100755 index 332310ab66..0000000000 --- a/doc/scripts/process-native-protocol-specs-in-docker.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/sh -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[ -f "../build.xml" ] || { echo "build.xml must exist (current directory needs to be doc/ in cassandra repo"; exit 1; } -[ -f "antora.yml" ] || { echo "antora.yml must exist (current directory needs to be doc/ in cassandra repo"; exit 1; } - -# Variables -GO_VERSION="1.23.1" -TMPDIR="${TMPDIR:-/tmp}" - -check_go_version() { - if command -v go &>/dev/null; then - local installed_version=$(go version | awk '{print $3}' | sed 's/go//') - if [ "$(printf '%s\n' "$GO_VERSION" "$installed_version" | sort -V | head -n1)" = "$GO_VERSION" ]; then - echo "Detected Go $installed_version (>= $GO_VERSION)" - return 0 - else - echo "Detected unsupported Go $installed_version (< $GO_VERSION), please update to supported version." - fi - else - echo "No Go installation detected, please install Go (>= $GO_VERSION)" - fi - return 1 -} - -if ! check_go_version; then - echo " Please install/upgrade Golang for 'ant gen-doc', or specify '-Dant.gen-doc.skip=true' to skip this step." - echo " For download and installation instructions see https://go.dev/doc/install" - exit 1 -fi - -# Step 1: Building the parser -echo "Building the cqlprotodoc..." -DIR="$(pwd)" -cd "${TMPDIR}" - -rm -rf "${TMPDIR}/cassandra-website" -git clone -n --depth=1 --filter=tree:0 https://github.com/apache/cassandra-website - -if [ $? != "0" ]; then - echo "Error occured while cloning https://github.com/apache/cassandra-website" - exit 1 -fi - -cd "${TMPDIR}/cassandra-website" -git sparse-checkout set --no-cone /cqlprotodoc -git checkout -cd "${TMPDIR}/cassandra-website/cqlprotodoc" -rm -rf "${TMPDIR}/cqlprotodoc" -go build -o "$TMPDIR"/cqlprotodoc - -# Step 2: Process the spec files using the parser -echo "Processing the .spec files..." -cd "${DIR}" -output_dir="modules/cassandra/attachments" -mkdir -p "${output_dir}" -"$TMPDIR"/cqlprotodoc . "${output_dir}" - -if ! ls ${output_dir}/native_protocol_v*.html > /dev/null 2>&1; then - echo "failed: No native_protocol_v*.html files generated in ${output_dir}" - exit 1 -fi - -# Step 4: Generate summary file -summary_file="modules/cassandra/pages/reference/native-protocol.adoc" - -# Write the header -echo "= Native Protocol Versions" > "$summary_file" -echo ":page-layout: default" >> "$summary_file" -echo >> "$summary_file" - -# Loop through the files from step 2 in reverse version order -for file in $(ls ${output_dir}/native_protocol_v*.html | sort -r | awk -F/ '{print $NF}'); do - version=$(echo "$file" | sed -E 's/native_protocol_v([0-9]+)\.html/\1/') - echo "== Native Protocol Version $version" >> "$summary_file" - echo >> "$summary_file" - echo "[source, html]" >> "$summary_file" - echo "++++" >> "$summary_file" - echo "include::cassandra:attachment\$$file[Version $version]" >> "$summary_file" - echo "++++" >> "$summary_file" - echo >> "$summary_file" -done - -# Navigation setup -echo "[source, js]" >> "$summary_file" -echo "++++" >> "$summary_file" -echo "" >> "$summary_file" - - -# Step 3: Cleanup - Remove the Cassandra and parser directories -echo "Cleaning up..." -cd "${DIR}" -rm -rf "${TMPDIR}/cassandra-website" "${TMPDIR}/cqlprotodoc" 2>/dev/null - -echo "Script completed successfully."