Automate topology transaction version compatibility

docs-main/appdev/deep-dives/external-signing-topology.mdx

@@ -987,9 +987,16 @@ The Canton console on a sequencer node of the target synchronizer also provides
 
 Each `Protocol Version` has a corresponding `Protobuf Version` for protobuf messages involved in the Canton protocol. That includes the `TopologyTransaction` message.
 
+{/* GENERATED_CANTON_TOPOLOGY_VERSIONS_START */}
+
+{/* GENERATED_FROM source="digital-asset/canton" ref="v3.5.15" asset="canton-open-source-3.5.15.tar.gz" digest="sha256:a11453d98917be6136cb2eaa3f800f5b67ee59ecff75907f7b59b4ccfd3766b8" protocol_version_count="2" */}
+
 | Protocol Version | Topology Transaction Protobuf Version |
-|------------------|---------------------------------------|
-| 34               | 30                                    |
+| --- | --- |
+| 34 | 30 |
+| 35 | 30 |
+
+{/* GENERATED_CANTON_TOPOLOGY_VERSIONS_END */}
 
 The protobuf version becomes relevant in the Version Wrapper section.
 

package.json

@@ -6,6 +6,7 @@
     "update:generated-reference-sources": "python3 scripts/update_generated_reference_sources.py",
     "generate:version-compatibility-dashboard": "python3 scripts/generate_network_component_versions.py",
     "generate:canton-topology-proto-link": "python3 scripts/generate_canton_topology_proto_link.py",
+    "generate:canton-topology-transaction-versions": "python3 scripts/generate_canton_topology_transaction_versions.py",
     "generate:json-api-reference": "python3 scripts/generate_json_api_reference.py",
     "generate:json-api-asyncapi-reference": "python3 scripts/generate_json_api_asyncapi_reference.py",
     "generate:grpc-ledger-api-reference": "python3 scripts/generate_grpc_ledger_api_reference.py",

scripts/canton_topology_transaction_versions.canton

@@ -0,0 +1,15 @@
+import io.circe.syntax._
+import com.digitalasset.canton.topology.transaction.TopologyTransaction
+import com.digitalasset.canton.version.ProtocolVersion
+
+val rows = ProtocolVersion.supported.filterNot(_.isDev).map { protocolVersion =>
+  protocolVersion.v.toString -> List(
+    TopologyTransaction.protoVersionFor(protocolVersion).v.toString
+  )
+}
+
+println(
+  io.circe.Json
+    .obj("topologyTransactionProtocolVersionToProtobufVersions" -> rows.asJson)
+    .spaces2
+)

scripts/generate_all_reference_docs.py

@@ -126,6 +126,11 @@ class ScriptJob:
         nav_slices=(),
         target_ids=(),
     ),
+    ScriptJob(
+        script_path=REPO_ROOT / "scripts" / "generate_canton_topology_transaction_versions.py",
+        nav_slices=(),
+        target_ids=(),
+    ),
 ]
 
 

scripts/generate_canton_topology_transaction_versions.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+from typing import cast
+
+from canton_release_reference import (
+    DEFAULT_RELEASE_REPO,
+    ReleaseAsset,
+    ensure_release_archive,
+    extract_release,
+    resolve_release_asset,
+    run_reference_script,
+)
+from docs_env import ensure_repo_direnv
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_CACHE_DIR = REPO_ROOT / ".internal" / "cache" / "canton-release-reference"
+DEFAULT_OUTPUT = (
+    REPO_ROOT / "docs-main" / "appdev" / "deep-dives" / "external-signing-topology.mdx"
+)
+REFERENCE_SCRIPT = REPO_ROOT / "scripts" / "canton_topology_transaction_versions.canton"
+GENERATED_START = "{/* GENERATED_CANTON_TOPOLOGY_VERSIONS_START */}"
+GENERATED_END = "{/* GENERATED_CANTON_TOPOLOGY_VERSIONS_END */}"
+TABLE_HEADER = "| Protocol Version | Topology Transaction Protobuf Version |"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Generate topology-transaction version mappings from a public Canton release binary."
+    )
+    parser.add_argument("--release-repo", default=DEFAULT_RELEASE_REPO)
+    parser.add_argument("--canton-tag")
+    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    parser.add_argument("--reference-json", type=Path)
+    parser.add_argument("--force-refresh", action="store_true")
+    parser.add_argument("--docs-json")
+    return parser.parse_args()
+
+
+def load_rows(payload: object) -> list[tuple[str, list[str]]]:
+    key = "topologyTransactionProtocolVersionToProtobufVersions"
+    if not isinstance(payload, dict) or not isinstance(payload.get(key), list):
+        raise ValueError(f"Canton reference JSON must contain a {key} list")
+    rows: list[tuple[str, list[str]]] = []
+    for index, row in enumerate(payload[key]):
+        if (
+            not isinstance(row, list)
+            or len(row) != 2
+            or not isinstance(row[0], str)
+            or not isinstance(row[1], list)
+            or not row[1]
+            or not all(isinstance(value, str) and value for value in row[1])
+        ):
+            raise ValueError(f"Topology-version row {index} is invalid")
+        rows.append((row[0], cast(list[str], row[1])))
+    return rows
+
+
+def render_table(rows: list[tuple[str, list[str]]], *, asset: ReleaseAsset) -> str:
+    lines = [
+        GENERATED_START,
+        "",
+        (
+            "{/* GENERATED_FROM "
+            f'source="{DEFAULT_RELEASE_REPO}" ref="{asset.tag}" asset="{asset.name}" '
+            f'digest="{asset.digest}" protocol_version_count="{len(rows)}" */}}'
+        ),
+        "",
+        TABLE_HEADER,
+        "| --- | --- |",
+    ]
+    lines.extend(
+        f"| {protocol_version} | {', '.join(protobuf_versions)} |"
+        for protocol_version, protobuf_versions in rows
+    )
+    lines.extend(["", GENERATED_END])
+    return "\n".join(lines)
+
+
+def replace_table(page: str, table: str) -> str:
+    if GENERATED_START in page:
+        start = page.index(GENERATED_START)
+        end = page.index(GENERATED_END, start) + len(GENERATED_END)
+        return page[:start].rstrip() + "\n\n" + table + "\n\n" + page[end:].lstrip()
+
+    start = page.index(TABLE_HEADER)
+    lines = page[start:].splitlines(keepends=True)
+    table_line_count = 0
+    for line in lines:
+        if not line.strip():
+            break
+        table_line_count += 1
+    end = start + sum(len(line) for line in lines[:table_line_count])
+    return page[:start].rstrip() + "\n\n" + table + "\n\n" + page[end:].lstrip()
+
+
+def main() -> int:
+    ensure_repo_direnv(
+        repo_root=REPO_ROOT, script_path=Path(__file__).resolve(), argv=sys.argv[1:]
+    )
+    args = parse_args()
+    asset = resolve_release_asset(release_repo=args.release_repo, tag=args.canton_tag)
+    if args.reference_json:
+        payload = json.loads(args.reference_json.read_text(encoding="utf-8"))
+    else:
+        archive_path = ensure_release_archive(
+            asset=asset, cache_dir=args.cache_dir, force_refresh=args.force_refresh
+        )
+        distribution_root = extract_release(
+            archive_path=archive_path,
+            asset=asset,
+            cache_dir=args.cache_dir,
+            force_refresh=args.force_refresh,
+        )
+        payload = run_reference_script(
+            distribution_root=distribution_root,
+            script_path=REFERENCE_SCRIPT,
+            cache_dir=args.cache_dir,
+            cache_namespace="topology-transaction-versions",
+            asset=asset,
+            force_refresh=args.force_refresh,
+        )
+
+    rows = load_rows(payload)
+    page = args.output.read_text(encoding="utf-8")
+    args.output.write_text(
+        replace_table(page, render_table(rows, asset=asset)), encoding="utf-8"
+    )
+    print(
+        f"Generated {len(rows)} topology/protobuf mappings from {asset.tag} in {args.output}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/update_generated_reference_prs.py

@@ -485,22 +485,27 @@ class UpdateTarget:
     ),
     UpdateTarget(
         key="canton-topology-proto-link",
-        title="Update Canton topology.proto link",
+        title="Update Canton topology references",
         branch="generated-docs/canton-topology-proto-link/update",
         description=(
             "Resolves the latest stable digital-asset/canton release, derives the matching "
-            "release-line branch URL for topology.proto, verifies the URL is reachable, and "
-            "updates the generated MDX export used by the topology reference page."
+            "release-line branch URL for topology.proto, and regenerates the topology-transaction "
+            "protocol/protobuf compatibility table from the public release binary."
         ),
         generate_commands=(
             ("nix-shell", "--run", "npm run generate:canton-topology-proto-link"),
+            ("nix-shell", "--run", "npm run generate:canton-topology-transaction-versions"),
+        ),
+        paths=(
+            "docs-main/snippets/generated/canton-topology-proto-link.mdx",
+            "docs-main/appdev/deep-dives/external-signing-topology.mdx",
         ),
-        paths=("docs-main/snippets/generated/canton-topology-proto-link.mdx",),
         summary_kind="static",
         summary_path=None,
         summary_label=None,
         validation=(
             "npm run generate:canton-topology-proto-link",
+            "npm run generate:canton-topology-transaction-versions",
             "git diff --check",
         ),
     ),

tests/test_canton_topology_transaction_versions.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+import unittest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+from scripts import generate_all_reference_docs  # noqa: E402
+from scripts import generate_canton_topology_transaction_versions as generator  # noqa: E402
+
+
+class CantonTopologyTransactionVersionsTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.asset = generator.ReleaseAsset(
+            tag="v3.5.15",
+            version="3.5.15",
+            name="canton-open-source-3.5.15.tar.gz",
+            url="https://github.com/digital-asset/canton/releases/download/v3.5.15/canton-open-source-3.5.15.tar.gz",
+            size=123,
+            digest="sha256:" + "a" * 64,
+        )
+
+    def test_uses_public_release_source(self) -> None:
+        self.assertEqual(generator.DEFAULT_RELEASE_REPO, "digital-asset/canton")
+        self.assertNotIn(
+            "DACH-NY", generator.REFERENCE_SCRIPT.read_text(encoding="utf-8")
+        )
+
+    def test_aggregate_generator_includes_topology_version_table(self) -> None:
+        job = next(
+            job
+            for job in generate_all_reference_docs.SCRIPT_JOBS
+            if job.script_path.name
+            == "generate_canton_topology_transaction_versions.py"
+        )
+        self.assertEqual(job.nav_slices, ())
+        self.assertEqual(job.target_ids, ())
+
+    def test_load_rows_validates_shape(self) -> None:
+        key = "topologyTransactionProtocolVersionToProtobufVersions"
+        self.assertEqual(
+            generator.load_rows({key: [["34", ["30"]], ["35", ["30"]]]}),
+            [("34", ["30"]), ("35", ["30"])],
+        )
+        with self.assertRaisesRegex(ValueError, "row 0"):
+            generator.load_rows({key: [["34", []]]})
+
+    def test_render_table_includes_all_rows_and_provenance(self) -> None:
+        table = generator.render_table(
+            [("34", ["30"]), ("35", ["30"])], asset=self.asset
+        )
+        self.assertIn('source="digital-asset/canton"', table)
+        self.assertIn('protocol_version_count="2"', table)
+        self.assertIn("| 34 | 30 |", table)
+        self.assertIn("| 35 | 30 |", table)
+
+    def test_replace_table_is_idempotent_and_preserves_page(self) -> None:
+        page = (
+            "Before\n\n"
+            "| Protocol Version | Topology Transaction Protobuf Version |\n"
+            "| --- | --- |\n"
+            "| 34 | 30 |\n\n"
+            "After\n"
+        )
+        table = generator.render_table(
+            [("34", ["30"]), ("35", ["30"])], asset=self.asset
+        )
+        first = generator.replace_table(page, table)
+        second = generator.replace_table(first, table)
+
+        self.assertEqual(first, second)
+        self.assertIn("Before", first)
+        self.assertIn("After", first)
+        self.assertEqual(first.count(generator.TABLE_HEADER), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_update_generated_reference_prs.py

@@ -375,6 +375,7 @@ def test_generated_clean_paths_include_target_paths_and_internal_output() -> Non
     assert "docs-main/global-synchronizer/reference/canton-console-commands.mdx" in clean_paths
     assert "docs-main/global-synchronizer/reference/error-codes.mdx" in clean_paths
     assert "docs-main/release-notes/releases-and-versioning.mdx" in clean_paths
+    assert "docs-main/appdev/deep-dives/external-signing-topology.mdx" in clean_paths
     assert "docs-main/global-synchronizer/reference/canton-metrics.mdx" in clean_paths
     assert "docs-main/global-synchronizer/release-notes" in clean_paths
     assert "docs-main/integrations/release-notes/wallet-gateway.mdx" in clean_paths