diff --git a/ci/test/check-docs-markdown.py b/ci/test/check-docs-markdown.py new file mode 100755 index 0000000000000..9a5902b905195 --- /dev/null +++ b/ci/test/check-docs-markdown.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +"""Report on the size and links of the agent-readable Markdown docs. + +Reads a built markdown-docs tree, the output of +`hugo --config config.toml,config.skill.toml`, and reports: + +- pages over the 50,000 and 100,000 character thresholds of the + Agent-Friendly Documentation Spec (https://agentdocsspec.com/spec/web/); +- links to another Markdown page, by the URL the page is served at, that name + no file in the tree; +- links that repeat the site's base path, or the /docs/ prefix, after it; +- root-relative links, such as `/sql/create-cluster`, and path-relative links, + such as `../create-role`, which do not resolve once the Markdown is read away + from the site. + +Links are read from inline links, reference definitions, and href and src +attributes, outside fenced code blocks. + +It also reports pages carrying Hugo shortcode syntax, such as "{{< tabs >}}", +outside fenced code blocks: a shortcode that reached the Markdown unrendered. + +Prints a summary and exits 1 if any page is over 100,000 characters, any +link is broken or relative, or any page carries shortcode syntax. With +--fail-on-links, only broken or relative links cause a failure, so known +oversized pages and unrendered shortcodes remain warnings in CI. + +Example usage: + + $ ci/test/check-docs-markdown.py doc/user/public/docs/markdown-docs \\ + https://materialize.com/docs/markdown-docs/ +""" + +import argparse +import re +import sys +from pathlib import Path +from urllib.parse import urlparse + +WARN_CHARS = 50_000 +FAIL_CHARS = 100_000 + +LINK_RES = ( + re.compile(r"\]\(([^)\s]+)"), + re.compile(r"^ {0,3}\[[^\]]+\]:[ \t]*([./]\S*)(?:[ \t]*$|[ \t]+[\"'(])"), + re.compile(r"(?:href|src)=\"([^\"]+)\""), +) +NOT_RELATIVE_RE = re.compile(r"^(#|//|<|[A-Za-z][A-Za-z0-9+.-]*:)") +SHORTCODE_RE = re.compile(r"\{\{[<%]") +FENCE_RE = re.compile(r"^(```|~~~)") + + +def prose_lines(text: str) -> list[str]: + """Return the lines of text outside fenced code blocks.""" + lines = [] + fenced = False + for line in text.splitlines(): + if FENCE_RE.match(line.lstrip()): + fenced = not fenced + continue + if not fenced: + lines.append(line) + return lines + + +def links(lines: list[str]) -> list[str]: + """Return the Markdown link targets in lines.""" + return [t for line in lines for link_re in LINK_RES for t in link_re.findall(line)] + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("root", type=Path, help="the built markdown-docs directory") + parser.add_argument( + "base_url", help="the URL the directory is served at, ending in a slash" + ) + parser.add_argument( + "--fail-on-links", + action="store_true", + help="fail only for broken or relative links", + ) + args = parser.parse_args() + + # A root-relative source link that still carries the base path, or the + # production /docs/ prefix, comes out with that prefix twice, as in + # /materialize/PR/materialize/PR/sql/... or /materialize/PR/docs/sql/... . + site_url = args.base_url.removesuffix("markdown-docs/") + doubled_prefixes = {f"{site_url}docs/"} + if base_path := urlparse(site_url).path.lstrip("/"): + doubled_prefixes.add(f"{site_url}{base_path}") + + pages = sorted(args.root.rglob("*.md")) + large: list[tuple[int, str]] = [] + broken: list[tuple[str, str]] = [] + doubled_prefix: list[tuple[str, str]] = [] + relative: dict[str, int] = {"root": 0, "path": 0} + relative_pages: set[str] = set() + shortcode_pages: list[str] = [] + for page in pages: + name = str(page.relative_to(args.root)) + text = page.read_text() + if len(text) > WARN_CHARS: + large.append((len(text), name)) + lines = prose_lines(text) + if any(SHORTCODE_RE.search(line) for line in lines): + shortcode_pages.append(name) + for target in links(lines): + if target.startswith(args.base_url): + path = target[len(args.base_url) :].split("#", 1)[0] + if not (args.root / path).is_file(): + broken.append((name, target)) + elif target.startswith(tuple(doubled_prefixes)): + doubled_prefix.append((name, target)) + elif not NOT_RELATIVE_RE.match(target): + relative["root" if target.startswith("/") else "path"] += 1 + relative_pages.add(name) + + over_fail = [entry for entry in large if entry[0] > FAIL_CHARS] + print(f"Markdown pages: {len(pages)}") + print(f"Over {WARN_CHARS:,} characters: {len(large)}") + print(f"Over {FAIL_CHARS:,} characters: {len(over_fail)}") + for chars, name in sorted(large, reverse=True): + print(f" {chars:>9,} {name}") + print(f"Broken links to Markdown pages: {len(broken)}") + for name, target in broken: + print(f" {name}: {target}") + print(f"Links with a repeated base path or /docs/ prefix: {len(doubled_prefix)}") + for name, target in doubled_prefix: + print(f" {name}: {target}") + print( + f"Relative links: {relative['root']} root-relative and {relative['path']}" + f" path-relative, on {len(relative_pages)} pages" + ) + print(f"Pages with unrendered shortcode syntax: {len(shortcode_pages)}") + for name in shortcode_pages: + print(f" {name}") + failed = bool(broken or doubled_prefix or relative_pages) + if not args.fail_on_links: + failed = failed or bool(over_fail or shortcode_pages) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/test/lint-docs.sh b/ci/test/lint-docs.sh index 250f03ad4c05c..464646ff18db3 100755 --- a/ci/test/lint-docs.sh +++ b/ci/test/lint-docs.sh @@ -15,9 +15,15 @@ set -euo pipefail . misc/shlib/shlib.bash -git clean -ffdX ci/www/public +git clean -ffdX ci/www/public ci/www/preview try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --destination ../../ci/www/public/docs try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/public/docs/markdown-docs +try python3 ci/test/test_check_docs_markdown.py +try python3 ci/test/check-docs-markdown.py ci/www/public/docs/markdown-docs https://ci.materialize.com/docs/markdown-docs/ --fail-on-links +# A base path other than /docs, like a preview's, catches link rewrites that +# confuse the base path with the production /docs/ prefix. +try hugo --gc --baseURL https://ci.materialize.com/materialize/lint --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/preview/markdown-docs +try python3 ci/test/check-docs-markdown.py ci/www/preview/markdown-docs https://ci.materialize.com/materialize/lint/markdown-docs/ --fail-on-links echo "" > ci/www/public/index.html try htmltest -s ci/www/public -c doc/user/.htmltest.yml try ci/test/lint-docs-catalog.sh diff --git a/ci/test/preview-docs.sh b/ci/test/preview-docs.sh index 8a122faf50a6f..939548eeecb2f 100755 --- a/ci/test/preview-docs.sh +++ b/ci/test/preview-docs.sh @@ -23,8 +23,9 @@ cd doc/user # Build main docs to public/ hugo --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" -# Build skill docs to public/markdown-docs/ -hugo --config config.toml,config.skill.toml --gc --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy +# Build skill docs to public/markdown-docs/. --environment preview makes their +# absolute links point at preview.materialize.com, like the HTML build above. +hugo --config config.toml,config.skill.toml --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy cat > config.deployment.toml < None: + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) / "markdown-docs" + self.root.mkdir() + self.page = self.root / "index.md" + + def check( + self, content: str, *options: str, base_url: str = BASE_URL + ) -> subprocess.CompletedProcess[str]: + self.page.write_text(content) + return subprocess.run( + [sys.executable, str(CHECKER), str(self.root), base_url, *options], + capture_output=True, + text=True, + check=False, + ) + + def test_known_size_and_shortcode_warnings_do_not_fail_link_gate(self) -> None: + content = "# Example\n{{< tabs >}}\n" + "x" * 100_000 + self.assertEqual(self.check(content, "--fail-on-links").returncode, 0) + self.assertEqual(self.check(content).returncode, 1) + + def test_missing_markdown_page_fails_link_gate(self) -> None: + result = self.check(f"[Missing]({BASE_URL}missing/index.md)", "--fail-on-links") + self.assertEqual(result.returncode, 1) + self.assertIn("Broken links to Markdown pages: 1", result.stdout) + + def test_relative_link_fails_link_gate(self) -> None: + result = self.check("[Relative](/sql/create-cluster/)", "--fail-on-links") + self.assertEqual(result.returncode, 1) + self.assertIn("Relative links: 1 root-relative", result.stdout) + + def test_doubled_docs_prefix_fails_link_gate(self) -> None: + bad_url = BASE_URL.removesuffix("markdown-docs/") + "docs/sql/create-cluster/" + result = self.check(f"[Malformed]({bad_url})", "--fail-on-links") + self.assertEqual(result.returncode, 1) + self.assertIn( + "Links with a repeated base path or /docs/ prefix: 1", result.stdout + ) + self.assertIn(bad_url, result.stdout) + + def test_repeated_base_path_fails_link_gate(self) -> None: + site_url = BASE_URL.removesuffix("markdown-docs/") + bad_url = site_url + "materialize/39470/sql/types/numeric" + result = self.check(f"[Malformed]({bad_url})", "--fail-on-links") + self.assertEqual(result.returncode, 1) + self.assertIn( + "Links with a repeated base path or /docs/ prefix: 1", result.stdout + ) + self.assertIn(bad_url, result.stdout) + + def test_site_at_root_flags_no_link_as_repeated(self) -> None: + result = self.check( + "[Page](https://example.com/sql/types/numeric)", + "--fail-on-links", + base_url="https://example.com/markdown-docs/", + ) + self.assertEqual(result.returncode, 0, result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/doc/user/config.toml b/doc/user/config.toml index 213219ed88f7b..d0886400f28bf 100644 --- a/doc/user/config.toml +++ b/doc/user/config.toml @@ -136,6 +136,13 @@ unsafe = true startLevel = 2 endLevel = 4 +# hugo deploy sets each file's Content-Type from the first Hugo media type with +# its suffix, and for ".xml" that is RSS, so the sitemap was served as +# application/rss+xml. Matchers are tried in order; the first match wins. +[[deployment.matchers]] +pattern = "^(.+/)?sitemap\\.xml$" +contentType = "application/xml" + [[deployment.targets]] name = "production" url = "s3://materialize-website?region=us-east-1" diff --git a/doc/user/content/fundamentals/concepts/reaction-time.md b/doc/user/content/fundamentals/concepts/reaction-time.md index 563ab52f873e6..5dbc4443f0f08 100644 --- a/doc/user/content/fundamentals/concepts/reaction-time.md +++ b/doc/user/content/fundamentals/concepts/reaction-time.md @@ -1,6 +1,6 @@ --- title: Reaction Time, Freshness, and Query Latency -description: "Learn about indexes in Materialize." +description: "Learn about freshness, query latency, and reaction time, and how they determine how current and how fast query results are in Materialize." menu: main: parent: concepts diff --git a/doc/user/content/sql/types/bytea.md b/doc/user/content/sql/types/bytea.md index e96ae5023fc35..91b3f6afda973 100644 --- a/doc/user/content/sql/types/bytea.md +++ b/doc/user/content/sql/types/bytea.md @@ -1,6 +1,6 @@ --- title: "bytea type" -description: "Expresses a Unicode string" +description: "Expresses a binary string" aliases: - /sql/types/string - /sql/types/varchar diff --git a/doc/user/layouts/_default/baseof.html b/doc/user/layouts/_default/baseof.html index 167df07159dfe..c5667790d88df 100644 --- a/doc/user/layouts/_default/baseof.html +++ b/doc/user/layouts/_default/baseof.html @@ -32,6 +32,20 @@ } + {{- /* + A pointer for agents that read the HTML, as the Agent-Friendly + Documentation Spec recommends: hidden on screen and from assistive + technology, and before the navigation so that agents reach it first. + */}} + {{- if partial "has-markdown.html" . }} + + {{- end }} + {{ partial "header.html" . }} {{ block "body" .}}
{{ partial "sidebar.html" . }} diff --git a/doc/user/layouts/_default/list.skill.md b/doc/user/layouts/_default/list.skill.md index fc54862729fb7..b5f1d21211af9 100644 --- a/doc/user/layouts/_default/list.skill.md +++ b/doc/user/layouts/_default/list.skill.md @@ -6,15 +6,24 @@ {{ .Description }} {{ end }} -{{ .RenderShortcodes }} +{{ partial "markdown-body.html" (dict "page" .) }} -{{ range .Pages }} -{{- if not (in $excludedSections .Section) }} ---- +{{- /* + List the pages below this one rather than embedding their content. Each of + them is published as its own Markdown file, and embedding them made section + pages such as sql/index.md several hundred KB, too long for an agent to read. +*/ -}} +{{- $children := slice -}} +{{- range .Pages -}} + {{- if and .RelPermalink (not (in $excludedSections .Section)) -}} + {{- $children = $children | append . -}} + {{- end -}} +{{- end -}} +{{- with $children }} -## {{ .Title }} +## Pages in this section -{{ .RenderShortcodes }} -{{ end -}} +{{ range . }}- [{{ .Title }}]({{ partial "markdown-link-url.html" (dict "page" .) }}){{ with .Description }}{{ $d := trim (replaceRE `\s+` " " .) " " }}{{ if $d }}: {{ $d }}{{ end }}{{ end }} {{ end }} +{{- end }} {{- end -}} diff --git a/doc/user/layouts/_default/single.skill.md b/doc/user/layouts/_default/single.skill.md index c071d9ecf3f08..c6b20f7b3c8e1 100644 --- a/doc/user/layouts/_default/single.skill.md +++ b/doc/user/layouts/_default/single.skill.md @@ -6,5 +6,5 @@ {{ .Description }} {{- end -}} -{{- .RenderShortcodes -}} +{{- partial "markdown-body.html" (dict "page" .) -}} {{- end -}} diff --git a/doc/user/layouts/index.llmstxt.txt b/doc/user/layouts/index.llmstxt.txt index f1c320c6ed30a..9d93db0bce758 100644 --- a/doc/user/layouts/index.llmstxt.txt +++ b/doc/user/layouts/index.llmstxt.txt @@ -9,11 +9,17 @@ by the Apache License, Version 2.0. */ -}} {{- /* - Renders /llms.txt, an index of every docs page for LLM and agent consumers. - Format per https://llmstxt.org/: an H1, a `>` summary blockquote, a short - orienting note, then one H2 per top-level docs section listing + Renders /llms.txt, an index of the docs for LLM and agent consumers, and one +
/llms.txt per top-level docs section. Format per https://llmstxt.org/: + an H1, a `>` summary blockquote and a short orienting note. The root index + then lists the section indexes; each section index lists its pages as "- [Title](url): description" entries. + The pages are split across section indexes because one index of every page + passed 85,000 characters, over the Agent-Friendly Documentation Spec's + 50,000. The section indexes are published from this template, with + resources.FromString, so that all of llms.txt is built in one place. + Links point at each page's Markdown rendition rather than its HTML page, so a consumer fetches prose instead of site chrome. @@ -39,13 +45,6 @@ {{- /* build.render=never leaves a page with no permalink to link to. */ -}} {{- if not .RelPermalink -}}{{- $keep = false -}}{{- end -}} {{- if .Params.headless -}}{{- $keep = false -}}{{- end -}} - {{- /* - Per-version release notes, filtered by path rather than permalink: the - older ones predate the render:never convention and still render, so the - permalink check above catches only part of the set. The /releases/ landing - page stays. - */ -}} - {{- if strings.HasPrefix .Path "/releases/v" -}}{{- $keep = false -}}{{- end -}} {{- with .Params.robots -}} {{- if in (lower .) "noindex" -}}{{- $keep = false -}}{{- end -}} {{- end -}} @@ -115,17 +114,33 @@ {{- with site.GetPage "/headless/materialize-intro/intro" -}} {{- $intro = trim (replaceRE `\s+` " " .RawContent) " " -}} {{- end -}} +{{- $note := "Every link below points at the Markdown source of a documentation page. For\nthe HTML page instead, drop `markdown-docs/` from the URL and drop the\ntrailing `index.md`." -}} +{{- $sections := slice -}} +{{- range $groups -}} + {{- $items := sort (where $eligible "Section" .key) "RelPermalink" -}} + {{- if $items -}} + {{- $dir := .key | default (urlize .title) -}} + {{- $lines := slice (printf "# %s: %s\n\n> Pages in the %s section of the %s.\n\n%s\n" site.Title .title .title site.Title $note) -}} + {{- range $items -}} + {{- $line := printf "- [%s](%s)" .Title (partial "markdown-url.html" (dict "page" . "absolute" true)) -}} + {{- with .Description -}}{{- $d := trim (replaceRE `\s+` " " .) " " -}}{{- if $d -}}{{- $line = printf "%s: %s" $line $d -}}{{- end -}}{{- end -}} + {{- $lines = $lines | append $line -}} + {{- end -}} + {{- $index := resources.FromString (printf "%s/llms.txt" $dir) (printf "%s\n" (delimit $lines "\n")) -}} + {{- with $index.Publish -}}{{- end -}} + {{- $sections = $sections | append (dict "title" .title "url" (partial "llms-txt-url.html" (dict "section" $dir "absolute" true)) "count" (len $items)) -}} + {{- end -}} +{{- end -}} # {{ site.Title }} > {{ $intro }} -Every link below points at the Markdown source of a documentation page. For the -HTML page instead, drop `markdown-docs/` from the URL and drop the trailing -`index.md`. Per-version release notes are not listed individually; see the -Releases section. -{{ range $groups }}{{ $items := sort (where $eligible "Section" .key) "RelPermalink" }}{{ if $items }} -## {{ .title }} +Each link below points at the index of one documentation section. A section +index lists every page in the section, with a link to its Markdown source and a +one-line description. + +## Sections -{{ range $items }}- [{{ .Title }}]({{ partial "markdown-url.html" (dict "page" . "absolute" true) }}){{ with .Description }}{{ $d := trim (replaceRE `\s+` " " .) " " }}{{ if $d }}: {{ $d }}{{ end }}{{ end }} -{{ end }}{{ end }}{{ end -}} +{{ range $sections }}- [{{ .title }}]({{ .url }}): {{ .count }} {{ cond (eq .count 1) "page" "pages" }} +{{ end }} {{- end -}} diff --git a/doc/user/layouts/partials/absolute-permalink.html b/doc/user/layouts/partials/absolute-permalink.html new file mode 100644 index 0000000000000..ad1f8fb3ca1a5 --- /dev/null +++ b/doc/user/layouts/partials/absolute-permalink.html @@ -0,0 +1,13 @@ +{{- /* + Returns a page's permalink as an absolute URL. With a bare-path baseURL, such + as "/docs", .Permalink has no scheme or host, so take them from the + site-origin partial. + + Usage: + {{ partial "absolute-permalink.html" . }} +*/ -}} +{{- $url := .Permalink -}} +{{- if not (urls.Parse $url).Host -}} + {{- $url = printf "%s%s" (partial "site-origin.html" .) $url -}} +{{- end -}} +{{- return $url -}} diff --git a/doc/user/layouts/partials/has-markdown.html b/doc/user/layouts/partials/has-markdown.html new file mode 100644 index 0000000000000..7da8732d6dfa8 --- /dev/null +++ b/doc/user/layouts/partials/has-markdown.html @@ -0,0 +1,9 @@ +{{- /* + Returns whether a page has a Markdown rendition on the website. The 404 page + has none, and self-managed-docs/* branches build the HTML alone (see + ci/deploy_website/website.sh). + + Usage: + {{ if partial "has-markdown.html" . }}...{{ end }} +*/ -}} +{{- return and (ne .Kind "404") (not (in (urls.Parse site.BaseURL).Path "self-managed")) -}} diff --git a/doc/user/layouts/partials/head.html b/doc/user/layouts/partials/head.html index 623c07598169b..ca8664c028451 100644 --- a/doc/user/layouts/partials/head.html +++ b/doc/user/layouts/partials/head.html @@ -14,6 +14,9 @@ {{ end }} {{ $title }} +{{ if partial "has-markdown.html" . }} + +{{ end }} /markdown-docs/, where such links do not resolve. + + The pointer is the llms.txt directive the Agent-Friendly Documentation Spec + recommends. The skill bundle has no llms.txt, so it gets no pointer. + + In the website build, each link in an inline link, a reference definition + or an HTML href or src attribute is rewritten by markdown-link-target. The + bin/gen-claude-skill bundle has base path "/", and its links stay as written, + because there they name directories under the bundle root. + + Arguments (dict): + page (required) the Page whose body to render + + Usage: + {{ partial "markdown-body.html" (dict "page" .) }} +*/ -}} +{{- /* + Hugo heading attributes, such as "## Object Introspection (SHOW) { #show }", + are syntax for the HTML pass. Drop them from the Markdown, in both builds. +*/ -}} +{{- $content := replaceRE `(?m)^(#{1,6} .*?)[ \t]*\{[ \t]*[#.][^}\n]*\}[ \t]*$` "$1" .page.RenderShortcodes -}} +{{- $basePath := strings.TrimSuffix "/" (urls.Parse site.BaseURL).Path -}} +{{- if $basePath -}} + {{- /* + The leading blank line separates the pointer from the description that + single.skill.md writes just before the body without a newline; without it + the pointer joins the description's line and is not a blockquote. + */ -}} + {{- $content = printf "\n\n> For the complete documentation index, see [llms.txt](%s).\n\n%s" (partial "llms-txt-url.html" (dict "absolute" true)) $content -}} + {{- /* + Three link forms, each as (prefix, target, end): inline links, reference + definitions, and href or src attributes in HTML that shortcodes emit. A + reference definition's target must start with "/" or "." and end the line + or precede a title, so that prose such as "[Note]: Materialize ..." is not + mistaken for one. + */ -}} + {{- $forms := slice + (dict "find" `(\]\()([^)\s]+)([)\s])` "end" `[)\s]`) + (dict "find" `(?m)(^[ \t]{0,3}\[[^\]\n]+\]:[ \t]*)([./]\S*)([ \t]*$|[ \t]+["'(])` "end" `(?:[ \t]*$|[ \t]+["'(])`) + (dict "find" `((?:href|src)=")([^"]+)(")` "end" `"`) + -}} + {{- range $forms -}} + {{- $end := .end -}} + {{- range findRESubmatch .find $content -}} + {{- $prefix := index . 1 -}} + {{- $target := index . 2 -}} + {{- $url := partial "markdown-link-target.html" (dict "page" $.page "target" $target) -}} + {{- if $url -}} + {{- $quoted := replaceRE `([\\.+*?()|\[\]{}^$])` `\$1` (printf "%s%s" $prefix $target) -}} + {{- $content = replaceRE (printf `(?m)%s(%s)` $quoted $end) (printf "%s%s$1" $prefix $url) $content -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- return $content -}} diff --git a/doc/user/layouts/partials/markdown-link-target.html b/doc/user/layouts/partials/markdown-link-target.html new file mode 100644 index 0000000000000..af6fb56b7f478 --- /dev/null +++ b/doc/user/layouts/partials/markdown-link-target.html @@ -0,0 +1,41 @@ +{{- /* + Returns the absolute URL a link in a page's Markdown should point at in the + website build, or "" to leave the link as written. + + A link to a page becomes the absolute URL of that page's Markdown, keeping + any #fragment. Any other link inside the docs, such as an image, becomes an + absolute URL under the base path. Root-relative links ("/sql/create-cluster/") + resolve against the docs root, whether or not they already carry the base path + or the production "/docs/" prefix. Path-relative links ("../create-role") + resolve against the page's own directory, as they do from the HTML page. + Fragments, absolute URLs, other schemes and targets are left + alone. + + Arguments (dict): + page (required) the Page the link appears on + target (required) the link target as written + + Usage: + {{ partial "markdown-link-target.html" (dict "page" . "target" "../create-role") }} +*/ -}} +{{- $url := "" -}} +{{- $target := .target -}} +{{- if not (or (strings.HasPrefix $target "#") (strings.HasPrefix $target "//") (strings.HasPrefix $target "<") (findRE `^[A-Za-z][A-Za-z0-9+.-]*:` $target)) -}} + {{- $basePath := strings.TrimSuffix "/" (urls.Parse site.BaseURL).Path -}} + {{- $path := index (split $target "#") 0 -}} + {{- $fragment := strings.TrimPrefix $path $target -}} + {{- if strings.HasPrefix $path "/" -}} + {{- /* Shortcode links already carry the base path from relURL, and data-file links can carry the production /docs/ prefix in any build. Leaving either in place repeats a prefix in the URL, which 404s. */ -}} + {{- $path = strings.TrimPrefix (printf "%s/" $basePath) $path | strings.TrimPrefix "/docs/" | strings.TrimPrefix "/" | printf "/%s" -}} + {{- else -}} + {{- $dir := .page.RelPermalink | strings.TrimPrefix $basePath | strings.TrimSuffix "index.md" -}} + {{- $path = path.Join $dir $path | printf "/%s" | replaceRE `^//` "/" -}} + {{- end -}} + {{- $url = printf "%s%s%s%s" (partial "site-origin.html" .) $basePath $path $fragment -}} + {{- if not (path.Ext $path) -}} + {{- with site.GetPage (strings.TrimSuffix "/" $path) -}} + {{- $url = printf "%s%s" (partial "markdown-url.html" (dict "page" . "absolute" true)) $fragment -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- return $url -}} diff --git a/doc/user/layouts/partials/markdown-link-url.html b/doc/user/layouts/partials/markdown-link-url.html new file mode 100644 index 0000000000000..d1f8d8ef52c24 --- /dev/null +++ b/doc/user/layouts/partials/markdown-link-url.html @@ -0,0 +1,21 @@ +{{- /* + Returns the URL that a link to a page should use inside the Markdown output. + + The Markdown output is built twice. The website build (ci/deploy_website/website.sh) + passes a --baseURL with a path, such as "/docs", and publishes the result at + /markdown-docs/, so a link there must be an absolute URL to the + target's Markdown rendition. bin/gen-claude-skill passes no --baseURL, so the + base path is "/", and the result is a skill bundle in which a root-relative + path such as "/sql/create-source/" names a directory under the bundle root. + + Arguments (dict): + page (required) the Page to link to + + Usage: + {{ partial "markdown-link-url.html" (dict "page" .) }} +*/ -}} +{{- $url := .page.RelPermalink -}} +{{- if ne (urls.Parse site.BaseURL).Path "/" -}} + {{- $url = partial "markdown-url.html" (dict "page" .page "absolute" true) -}} +{{- end -}} +{{- return $url -}} diff --git a/doc/user/layouts/partials/markdown-url.html b/doc/user/layouts/partials/markdown-url.html index 27890bbd1ed2a..c179b0a3fcd80 100644 --- a/doc/user/layouts/partials/markdown-url.html +++ b/doc/user/layouts/partials/markdown-url.html @@ -8,10 +8,8 @@ absolute (optional) prefix the URL with scheme://host baseURL is a bare path in the production and preview builds ("/docs", - "/materialize/$PR"), so .Permalink cannot yield an absolute URL. When baseURL - carries no host of its own, fall back to params.canonicalHost, except under - `hugo --environment preview`, where the artifacts serve from - preview.materialize.com. + "/materialize/$PR"), so .Permalink cannot yield an absolute URL. An absolute + URL takes its scheme and host from the site-origin partial. Usage: {{ partial "markdown-url.html" (dict "page" .) }} @@ -19,16 +17,13 @@ */ -}} {{- $basePath := (urls.Parse site.BaseURL).Path -}} {{- if not (strings.HasSuffix $basePath "/") -}}{{- $basePath = printf "%s/" $basePath -}}{{- end -}} -{{- $pagePath := .page.RelPermalink | strings.TrimPrefix $basePath -}} +{{- /* + In the Markdown pass, RelPermalink already ends in the skill format's + index.md; drop it so the URL is the same from either pass. +*/ -}} +{{- $pagePath := .page.RelPermalink | strings.TrimPrefix $basePath | strings.TrimSuffix "index.md" -}} {{- $url := printf "%smarkdown-docs/%sindex.md" $basePath $pagePath -}} {{- if .absolute -}} - {{- $parsed := urls.Parse site.BaseURL -}} - {{- $origin := site.Params.canonicalHost -}} - {{- if $parsed.Host -}} - {{- $origin = printf "%s://%s" $parsed.Scheme $parsed.Host -}} - {{- else if eq hugo.Environment "preview" -}} - {{- $origin = "https://preview.materialize.com" -}} - {{- end -}} - {{- $url = printf "%s%s" $origin $url -}} + {{- $url = printf "%s%s" (partial "site-origin.html" .) $url -}} {{- end -}} {{- return $url -}} diff --git a/doc/user/layouts/partials/site-origin.html b/doc/user/layouts/partials/site-origin.html new file mode 100644 index 0000000000000..2e8c6ea6108c6 --- /dev/null +++ b/doc/user/layouts/partials/site-origin.html @@ -0,0 +1,21 @@ +{{- /* + Returns the scheme and host the docs are served from, such as + "https://materialize.com", for building absolute URLs. + + baseURL is a bare path in the production and preview builds ("/docs", + "/materialize/$PR"), so .Permalink cannot yield an absolute URL. When baseURL + carries no host of its own, fall back to params.canonicalHost, except under + `hugo --environment preview`, where the artifacts serve from + preview.materialize.com. + + Usage: + {{ partial "site-origin.html" . }} +*/ -}} +{{- $parsed := urls.Parse site.BaseURL -}} +{{- $origin := site.Params.canonicalHost -}} +{{- if $parsed.Host -}} + {{- $origin = printf "%s://%s" $parsed.Scheme $parsed.Host -}} +{{- else if eq hugo.Environment "preview" -}} + {{- $origin = "https://preview.materialize.com" -}} +{{- end -}} +{{- return $origin -}} diff --git a/doc/user/layouts/shortcodes/include-example.skill.md b/doc/user/layouts/shortcodes/include-example.skill.md index 9227d8dcbb3c9..83b0872eea212 100644 --- a/doc/user/layouts/shortcodes/include-example.skill.md +++ b/doc/user/layouts/shortcodes/include-example.skill.md @@ -8,7 +8,7 @@ {{- $indent := .Get "indent" -}} {{- range $data }} {{- if eq .name $example -}} -{{- .description -}} +{{- with .description -}}{{ . }}{{- end -}} {{- if .code -}} {{- $code := .code -}} {{- if $indent }} diff --git a/doc/user/layouts/sitemap.xml b/doc/user/layouts/sitemap.xml new file mode 100644 index 0000000000000..ed1ff60301e87 --- /dev/null +++ b/doc/user/layouts/sitemap.xml @@ -0,0 +1,40 @@ +{{- /* + Copyright Materialize, Inc. and contributors. All rights reserved. + + Use of this software is governed by the Business Source License + included in the LICENSE file at the root of this repository. + + As of the Change Date specified in that file, in accordance with + the Business Source License, use of this software will be governed + by the Apache License, Version 2.0. +*/ -}} +{{- /* + Hugo's built-in sitemap template, from v0.152.2, with absolute URLs. The + sitemap protocol requires a full URL in every , and the built-in + template emits .Permalink, which has no host under a bare-path baseURL such + as "/docs". +*/ -}} +{{ printf "" | safeHTML }} + + {{ range where .Pages "Sitemap.Disable" "ne" true }} + {{- if .Permalink -}} + + {{ partial "absolute-permalink.html" . }}{{ if not .Lastmod.IsZero }} + {{ safeHTML ( .Lastmod.Format "2006-01-02T15:04:05-07:00" ) }}{{ end }}{{ with .Sitemap.ChangeFreq }} + {{ . }}{{ end }}{{ if ge .Sitemap.Priority 0.0 }} + {{ .Sitemap.Priority }}{{ end }}{{ if .IsTranslated }}{{ range .Translations }} + {{ end }} + {{ end }} + + {{- end -}} + {{ end }} +