doc: make the docs friendlier to coding agents (agent docs 1-7)

ci/test/check-docs-markdown.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+"""Report on the size and links of the agent-readable Markdown docs.
+
+Reads a built markdown-docs tree, the output of
+`hugo --config config.toml,config.skill.toml`, and reports:
+
+- pages over the 50,000 and 100,000 character thresholds of the
+  Agent-Friendly Documentation Spec (https://agentdocsspec.com/spec/web/);
+- links to another Markdown page, by the URL the page is served at, that name
+  no file in the tree;
+- links that repeat the site's base path, or the /docs/ prefix, after it;
+- root-relative links, such as `/sql/create-cluster`, and path-relative links,
+  such as `../create-role`, which do not resolve once the Markdown is read away
+  from the site.
+
+Links are read from inline links, reference definitions, and href and src
+attributes, outside fenced code blocks.
+
+It also reports pages carrying Hugo shortcode syntax, such as "{{< tabs >}}",
+outside fenced code blocks: a shortcode that reached the Markdown unrendered.
+
+Prints a summary and exits 1 if any page is over 100,000 characters, any
+link is broken or relative, or any page carries shortcode syntax. With
+--fail-on-links, only broken or relative links cause a failure, so known
+oversized pages and unrendered shortcodes remain warnings in CI.
+
+Example usage:
+
+    $ ci/test/check-docs-markdown.py doc/user/public/docs/markdown-docs \\
+        https://materialize.com/docs/markdown-docs/
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+WARN_CHARS = 50_000
+FAIL_CHARS = 100_000
+
+LINK_RES = (
+    re.compile(r"\]\(([^)\s]+)"),
+    re.compile(r"^ {0,3}\[[^\]]+\]:[ \t]*([./]\S*)(?:[ \t]*$|[ \t]+[\"'(])"),
+    re.compile(r"(?:href|src)=\"([^\"]+)\""),
+)
+NOT_RELATIVE_RE = re.compile(r"^(#|//|<|[A-Za-z][A-Za-z0-9+.-]*:)")
+SHORTCODE_RE = re.compile(r"\{\{[<%]")
+FENCE_RE = re.compile(r"^(```|~~~)")
+
+
+def prose_lines(text: str) -> list[str]:
+    """Return the lines of text outside fenced code blocks."""
+    lines = []
+    fenced = False
+    for line in text.splitlines():
+        if FENCE_RE.match(line.lstrip()):
+            fenced = not fenced
+            continue
+        if not fenced:
+            lines.append(line)
+    return lines
+
+
+def links(lines: list[str]) -> list[str]:
+    """Return the Markdown link targets in lines."""
+    return [t for line in lines for link_re in LINK_RES for t in link_re.findall(line)]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("root", type=Path, help="the built markdown-docs directory")
+    parser.add_argument(
+        "base_url", help="the URL the directory is served at, ending in a slash"
+    )
+    parser.add_argument(
+        "--fail-on-links",
+        action="store_true",
+        help="fail only for broken or relative links",
+    )
+    args = parser.parse_args()
+
+    # A root-relative source link that still carries the base path, or the
+    # production /docs/ prefix, comes out with that prefix twice, as in
+    # /materialize/PR/materialize/PR/sql/... or /materialize/PR/docs/sql/... .
+    site_url = args.base_url.removesuffix("markdown-docs/")
+    doubled_prefixes = {f"{site_url}docs/"}
+    if base_path := urlparse(site_url).path.lstrip("/"):
+        doubled_prefixes.add(f"{site_url}{base_path}")
+
+    pages = sorted(args.root.rglob("*.md"))
+    large: list[tuple[int, str]] = []
+    broken: list[tuple[str, str]] = []
+    doubled_prefix: list[tuple[str, str]] = []
+    relative: dict[str, int] = {"root": 0, "path": 0}
+    relative_pages: set[str] = set()
+    shortcode_pages: list[str] = []
+    for page in pages:
+        name = str(page.relative_to(args.root))
+        text = page.read_text()
+        if len(text) > WARN_CHARS:
+            large.append((len(text), name))
+        lines = prose_lines(text)
+        if any(SHORTCODE_RE.search(line) for line in lines):
+            shortcode_pages.append(name)
+        for target in links(lines):
+            if target.startswith(args.base_url):
+                path = target[len(args.base_url) :].split("#", 1)[0]
+                if not (args.root / path).is_file():
+                    broken.append((name, target))
+            elif target.startswith(tuple(doubled_prefixes)):
+                doubled_prefix.append((name, target))
+            elif not NOT_RELATIVE_RE.match(target):
+                relative["root" if target.startswith("/") else "path"] += 1
+                relative_pages.add(name)
+
+    over_fail = [entry for entry in large if entry[0] > FAIL_CHARS]
+    print(f"Markdown pages: {len(pages)}")
+    print(f"Over {WARN_CHARS:,} characters: {len(large)}")
+    print(f"Over {FAIL_CHARS:,} characters: {len(over_fail)}")
+    for chars, name in sorted(large, reverse=True):
+        print(f"  {chars:>9,}  {name}")
+    print(f"Broken links to Markdown pages: {len(broken)}")
+    for name, target in broken:
+        print(f"  {name}: {target}")
+    print(f"Links with a repeated base path or /docs/ prefix: {len(doubled_prefix)}")
+    for name, target in doubled_prefix:
+        print(f"  {name}: {target}")
+    print(
+        f"Relative links: {relative['root']} root-relative and {relative['path']}"
+        f" path-relative, on {len(relative_pages)} pages"
+    )
+    print(f"Pages with unrendered shortcode syntax: {len(shortcode_pages)}")
+    for name in shortcode_pages:
+        print(f"  {name}")
+    failed = bool(broken or doubled_prefix or relative_pages)
+    if not args.fail_on_links:
+        failed = failed or bool(over_fail or shortcode_pages)
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/test/lint-docs.sh

@@ -15,9 +15,15 @@ set -euo pipefail
 
 . misc/shlib/shlib.bash
 
-git clean -ffdX ci/www/public
+git clean -ffdX ci/www/public ci/www/preview
 try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --destination ../../ci/www/public/docs
 try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/public/docs/markdown-docs
+try python3 ci/test/test_check_docs_markdown.py
+try python3 ci/test/check-docs-markdown.py ci/www/public/docs/markdown-docs https://ci.materialize.com/docs/markdown-docs/ --fail-on-links
+# A base path other than /docs, like a preview's, catches link rewrites that
+# confuse the base path with the production /docs/ prefix.
+try hugo --gc --baseURL https://ci.materialize.com/materialize/lint --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/preview/markdown-docs
+try python3 ci/test/check-docs-markdown.py ci/www/preview/markdown-docs https://ci.materialize.com/materialize/lint/markdown-docs/ --fail-on-links
 echo "<!doctype html>" > ci/www/public/index.html
 try htmltest -s ci/www/public -c doc/user/.htmltest.yml
 try ci/test/lint-docs-catalog.sh

ci/test/preview-docs.sh

@@ -23,8 +23,9 @@ cd doc/user
 # Build main docs to public/
 hugo --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST"
 
-# Build skill docs to public/markdown-docs/
-hugo --config config.toml,config.skill.toml --gc --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy
+# Build skill docs to public/markdown-docs/. --environment preview makes their
+# absolute links point at preview.materialize.com, like the HTML build above.
+hugo --config config.toml,config.skill.toml --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy
 
 cat > config.deployment.toml <<EOF
 [[deployment.targets]]
@@ -44,3 +45,11 @@ curl -fsSL \
         \"target_url\": \"https://preview.materialize.com/materialize/$BUILDKITE_PULL_REQUEST/\",\
         \"context\": \"preview-docs\"\
     }"
+
+# Report on the preview's Markdown. The check does not fail the build.
+preview_url="https://preview.materialize.com/materialize/$BUILDKITE_PULL_REQUEST"
+
+ci_uncollapsed_heading "Checking the Markdown docs' sizes and links"
+if ! ../../ci/test/check-docs-markdown.py public/markdown-docs "$preview_url/markdown-docs/"; then
+    echo "check-docs-markdown found problems; see above. This check does not fail the build."
+fi

ci/test/test_check_docs_markdown.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+"""Exercise the Markdown link gate against a small built-docs fixture."""
+
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+CHECKER = Path(__file__).with_name("check-docs-markdown.py")
+BASE_URL = "https://preview.materialize.com/materialize/39470/markdown-docs/"
+
+
+class MarkdownLinkGateTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name) / "markdown-docs"
+        self.root.mkdir()
+        self.page = self.root / "index.md"
+
+    def check(
+        self, content: str, *options: str, base_url: str = BASE_URL
+    ) -> subprocess.CompletedProcess[str]:
+        self.page.write_text(content)
+        return subprocess.run(
+            [sys.executable, str(CHECKER), str(self.root), base_url, *options],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+    def test_known_size_and_shortcode_warnings_do_not_fail_link_gate(self) -> None:
+        content = "# Example\n{{< tabs >}}\n" + "x" * 100_000
+        self.assertEqual(self.check(content, "--fail-on-links").returncode, 0)
+        self.assertEqual(self.check(content).returncode, 1)
+
+    def test_missing_markdown_page_fails_link_gate(self) -> None:
+        result = self.check(f"[Missing]({BASE_URL}missing/index.md)", "--fail-on-links")
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("Broken links to Markdown pages: 1", result.stdout)
+
+    def test_relative_link_fails_link_gate(self) -> None:
+        result = self.check("[Relative](/sql/create-cluster/)", "--fail-on-links")
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("Relative links: 1 root-relative", result.stdout)
+
+    def test_doubled_docs_prefix_fails_link_gate(self) -> None:
+        bad_url = BASE_URL.removesuffix("markdown-docs/") + "docs/sql/create-cluster/"
+        result = self.check(f"[Malformed]({bad_url})", "--fail-on-links")
+        self.assertEqual(result.returncode, 1)
+        self.assertIn(
+            "Links with a repeated base path or /docs/ prefix: 1", result.stdout
+        )
+        self.assertIn(bad_url, result.stdout)
+
+    def test_repeated_base_path_fails_link_gate(self) -> None:
+        site_url = BASE_URL.removesuffix("markdown-docs/")
+        bad_url = site_url + "materialize/39470/sql/types/numeric"
+        result = self.check(f"[Malformed]({bad_url})", "--fail-on-links")
+        self.assertEqual(result.returncode, 1)
+        self.assertIn(
+            "Links with a repeated base path or /docs/ prefix: 1", result.stdout
+        )
+        self.assertIn(bad_url, result.stdout)
+
+    def test_site_at_root_flags_no_link_as_repeated(self) -> None:
+        result = self.check(
+            "[Page](https://example.com/sql/types/numeric)",
+            "--fail-on-links",
+            base_url="https://example.com/markdown-docs/",
+        )
+        self.assertEqual(result.returncode, 0, result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

doc/user/config.toml

@@ -136,6 +136,13 @@ unsafe = true
     startLevel = 2
     endLevel = 4
 
+# hugo deploy sets each file's Content-Type from the first Hugo media type with
+# its suffix, and for ".xml" that is RSS, so the sitemap was served as
+# application/rss+xml. Matchers are tried in order; the first match wins.
+[[deployment.matchers]]
+pattern = "^(.+/)?sitemap\\.xml$"
+contentType = "application/xml"
+
 [[deployment.targets]]
 name = "production"
 url = "s3://materialize-website?region=us-east-1"

doc/user/content/fundamentals/concepts/reaction-time.md

@@ -1,6 +1,6 @@
 ---
 title: Reaction Time, Freshness, and Query Latency
-description: "Learn about indexes in Materialize."
+description: "Learn about freshness, query latency, and reaction time, and how they determine how current and how fast query results are in Materialize."
 menu:
   main:
     parent: concepts

doc/user/content/sql/types/bytea.md

@@ -1,6 +1,6 @@
 ---
 title: "bytea type"
-description: "Expresses a Unicode string"
+description: "Expresses a binary string"
 aliases:
     - /sql/types/string
     - /sql/types/varchar

doc/user/layouts/_default/baseof.html

@@ -32,6 +32,20 @@
       }
     </script>
 
+    {{- /*
+      A pointer for agents that read the HTML, as the Agent-Friendly
+      Documentation Spec recommends: hidden on screen and from assistive
+      technology, and before the navigation so that agents reach it first.
+    */}}
+    {{- if partial "has-markdown.html" . }}
+    <div aria-hidden="true" style="position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap;">
+      For agents: an index of every documentation page is at
+      <a href="{{ partial "llms-txt-url.html" (dict) }}" tabindex="-1">llms.txt</a>,
+      and this page is available as
+      <a href="{{ partial "markdown-url.html" (dict "page" .) }}" tabindex="-1">Markdown</a>.
+    </div>
+    {{- end }}
+
     {{ partial "header.html" . }} {{ block "body" .}}
     <div class="content-wrapper">
       {{ partial "sidebar.html" . }}

doc/user/layouts/_default/list.skill.md

@@ -6,15 +6,24 @@
 {{ .Description }}
 {{ end }}
 
-{{ .RenderShortcodes }}
+{{ partial "markdown-body.html" (dict "page" .) }}
 
-{{ range .Pages }}
-{{- if not (in $excludedSections .Section) }}
----
+{{- /*
+  List the pages below this one rather than embedding their content. Each of
+  them is published as its own Markdown file, and embedding them made section
+  pages such as sql/index.md several hundred KB, too long for an agent to read.
+*/ -}}
+{{- $children := slice -}}
+{{- range .Pages -}}
+  {{- if and .RelPermalink (not (in $excludedSections .Section)) -}}
+    {{- $children = $children | append . -}}
+  {{- end -}}
+{{- end -}}
+{{- with $children }}
 
-## {{ .Title }}
+## Pages in this section
 
-{{ .RenderShortcodes }}
-{{ end -}}
+{{ range . }}- [{{ .Title }}]({{ partial "markdown-link-url.html" (dict "page" .) }}){{ with .Description }}{{ $d := trim (replaceRE `\s+` " " .) " " }}{{ if $d }}: {{ $d }}{{ end }}{{ end }}
 {{ end }}
+{{- end }}
 {{- end -}}

doc/user/layouts/_default/single.skill.md

@@ -6,5 +6,5 @@
 {{ .Description }}
 {{- end -}}
 
-{{- .RenderShortcodes -}}
+{{- partial "markdown-body.html" (dict "page" .) -}}
 {{- end -}}