|
| 1 | +package compat_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "fmt" |
| 5 | + "go/parser" |
| 6 | + "go/token" |
| 7 | + "io/fs" |
| 8 | + "os" |
| 9 | + "path/filepath" |
| 10 | + "regexp" |
| 11 | + "sort" |
| 12 | + "strings" |
| 13 | + "testing" |
| 14 | +) |
| 15 | + |
| 16 | +// citationRE matches `CONTRACT §N: "..."` in a comment, where N is one |
| 17 | +// or more digits and "..." is a double-quoted prose fragment. The |
| 18 | +// fragment may have been wrapped across two or more comment lines in |
| 19 | +// source — go/ast.CommentGroup.Text() joins consecutive comment lines |
| 20 | +// with single spaces before we run the regex, so the match is always |
| 21 | +// against a single logical line. |
| 22 | +// |
| 23 | +// Backtick-quoted fragments (e.g. `[]` for json) are deliberately |
| 24 | +// excluded: those are usually inline code references, not contract |
| 25 | +// quotations, and treating them as the latter would create noise. If a |
| 26 | +// contributor wants the harness to verify a quote, they write it with |
| 27 | +// double quotes. |
| 28 | +var citationRE = regexp.MustCompile(`CONTRACT §(\d+):\s*"([^"]+)"`) |
| 29 | + |
| 30 | +// sectionHeaderRE matches `## N. <title>` headers in CONTRACT.md and |
| 31 | +// captures the section number. |
| 32 | +var sectionHeaderRE = regexp.MustCompile(`(?m)^##\s+(\d+)\.\s+`) |
| 33 | + |
| 34 | +// minContractSections is the floor on how many top-level `## N. <title>` |
| 35 | +// sections CONTRACT.md should currently parse to. CONTRACT.md has |
| 36 | +// sections 1–8 today; a header refactor that drops below this floor |
| 37 | +// would silently let every cited section "not exist" and pass every |
| 38 | +// quote check vacuously. Bump this number when the contract grows. |
| 39 | +const minContractSections = 8 |
| 40 | + |
| 41 | +// citationProblem records one failing citation so the top-level test |
| 42 | +// can render all of them in a single diagnostic. |
| 43 | +type citationProblem struct { |
| 44 | + path string |
| 45 | + section string |
| 46 | + quote string |
| 47 | + reason string |
| 48 | +} |
| 49 | + |
| 50 | +// TestCONTRACTCitationsAreReal walks every .go file in the compat |
| 51 | +// module and, for every comment that quotes a section of the contract |
| 52 | +// in the form `CONTRACT §N`-colon-double-quoted-fragment, verifies the |
| 53 | +// fragment is a substring of section N of CONTRACT.md. A false quote |
| 54 | +// fails the test loudly so it cannot silently survive review. |
| 55 | +// |
| 56 | +// The test exists because two consecutive PRs landed comments |
| 57 | +// attributing prose to §5 that §5 does not contain (§5 is the Auth |
| 58 | +// section, with no language about hermeticity or `--help`). Go-side |
| 59 | +// review missed both. This guard is the structural fix: the next |
| 60 | +// fabricated quote dies in CI instead of in a post-merge incident |
| 61 | +// comment. |
| 62 | +func TestCONTRACTCitationsAreReal(t *testing.T) { |
| 63 | + // `go test` enters the package directory before running, so when |
| 64 | + // this file's tests execute the working directory is `compat/` and |
| 65 | + // CONTRACT.md lives one level up. If the compat module is ever |
| 66 | + // vendored into a different repo layout, this is the line that |
| 67 | + // needs updating. |
| 68 | + contractPath := filepath.Join("..", "CONTRACT.md") |
| 69 | + body, err := os.ReadFile(contractPath) |
| 70 | + if err != nil { |
| 71 | + t.Fatalf("read CONTRACT.md (looked at %s): %v", contractPath, err) |
| 72 | + } |
| 73 | + problems, err := findCitationProblems(".", string(body)) |
| 74 | + if err != nil { |
| 75 | + t.Fatalf("find citation problems: %v", err) |
| 76 | + } |
| 77 | + if len(problems) == 0 { |
| 78 | + return |
| 79 | + } |
| 80 | + t.Fatal(formatCitationProblems(problems)) |
| 81 | +} |
| 82 | + |
| 83 | +// findCitationProblems scans every .go file under root and returns the |
| 84 | +// list of `CONTRACT §N: "..."` citations whose quoted fragment is not |
| 85 | +// found in section N of contractMd. It is the test's load-bearing |
| 86 | +// logic, factored out so TestCitationsGuardCatchesFakeQuote can drive |
| 87 | +// it against a synthetic root + synthetic contract without touching |
| 88 | +// the real tree. |
| 89 | +func findCitationProblems(root, contractMd string) ([]citationProblem, error) { |
| 90 | + sections := splitContractSections(contractMd) |
| 91 | + if len(sections) < minContractSections { |
| 92 | + return nil, fmt.Errorf("parsed only %d `## N. <title>` sections from CONTRACT.md; expected at least %d. The header style may have changed — update sectionHeaderRE or bump minContractSections.", len(sections), minContractSections) |
| 93 | + } |
| 94 | + |
| 95 | + var problems []citationProblem |
| 96 | + walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { |
| 97 | + if err != nil { |
| 98 | + return err |
| 99 | + } |
| 100 | + if d.IsDir() { |
| 101 | + return nil |
| 102 | + } |
| 103 | + if !strings.HasSuffix(path, ".go") { |
| 104 | + return nil |
| 105 | + } |
| 106 | + fset := token.NewFileSet() |
| 107 | + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) |
| 108 | + if err != nil { |
| 109 | + return fmt.Errorf("parse %s: %w", path, err) |
| 110 | + } |
| 111 | + // Display path is the file path relative to root so error |
| 112 | + // messages stay short and meaningful when invoked with root=".". |
| 113 | + display := path |
| 114 | + if rel, relErr := filepath.Rel(root, path); relErr == nil { |
| 115 | + display = rel |
| 116 | + } |
| 117 | + for _, cg := range f.Comments { |
| 118 | + text := cg.Text() |
| 119 | + for _, m := range citationRE.FindAllStringSubmatch(text, -1) { |
| 120 | + section := m[1] |
| 121 | + quote := m[2] |
| 122 | + secBody, ok := sections[section] |
| 123 | + if !ok { |
| 124 | + problems = append(problems, citationProblem{display, section, quote, "no `## " + section + ". ` section exists in CONTRACT.md"}) |
| 125 | + continue |
| 126 | + } |
| 127 | + if !strings.Contains(normalize(secBody), normalize(quote)) { |
| 128 | + problems = append(problems, citationProblem{display, section, quote, "quoted text is not a substring of CONTRACT.md §" + section}) |
| 129 | + } |
| 130 | + } |
| 131 | + } |
| 132 | + return nil |
| 133 | + }) |
| 134 | + if walkErr != nil { |
| 135 | + return nil, walkErr |
| 136 | + } |
| 137 | + |
| 138 | + sort.Slice(problems, func(i, j int) bool { |
| 139 | + if problems[i].path != problems[j].path { |
| 140 | + return problems[i].path < problems[j].path |
| 141 | + } |
| 142 | + return problems[i].section < problems[j].section |
| 143 | + }) |
| 144 | + return problems, nil |
| 145 | +} |
| 146 | + |
| 147 | +// formatCitationProblems renders the problem list as the human-readable |
| 148 | +// failure message t.Fatal logs. Kept separate so the meta-test can |
| 149 | +// inspect both the structured list and the rendered string. |
| 150 | +func formatCitationProblems(problems []citationProblem) string { |
| 151 | + var b strings.Builder |
| 152 | + fmt.Fprintf(&b, "%d CONTRACT §X citation(s) do not match CONTRACT.md:\n\n", len(problems)) |
| 153 | + for _, p := range problems { |
| 154 | + fmt.Fprintf(&b, " %s\n", p.path) |
| 155 | + fmt.Fprintf(&b, " cite : CONTRACT §%s\n", p.section) |
| 156 | + fmt.Fprintf(&b, " quote: %q\n", p.quote) |
| 157 | + fmt.Fprintf(&b, " why : %s\n\n", p.reason) |
| 158 | + } |
| 159 | + b.WriteString("Fix by either dropping the quote or updating CONTRACT.md so §N actually contains the text.\n") |
| 160 | + b.WriteString("See compat/citations_test.go for the matching rule.") |
| 161 | + return b.String() |
| 162 | +} |
| 163 | + |
| 164 | +// splitContractSections returns a map from section number ("3") to the |
| 165 | +// raw markdown body of `## 3. <title>` up to (but not including) the |
| 166 | +// next `## N. <title>` header. The section header itself is included |
| 167 | +// in the body so that the title text is also citable. |
| 168 | +func splitContractSections(md string) map[string]string { |
| 169 | + idxs := sectionHeaderRE.FindAllStringSubmatchIndex(md, -1) |
| 170 | + out := map[string]string{} |
| 171 | + for i, m := range idxs { |
| 172 | + section := md[m[2]:m[3]] |
| 173 | + var end int |
| 174 | + if i+1 < len(idxs) { |
| 175 | + end = idxs[i+1][0] |
| 176 | + } else { |
| 177 | + end = len(md) |
| 178 | + } |
| 179 | + out[section] = md[m[0]:end] |
| 180 | + } |
| 181 | + return out |
| 182 | +} |
| 183 | + |
| 184 | +// normalize collapses runs of whitespace to single spaces so a quote |
| 185 | +// that was wrapped across two source-comment lines (joined with one |
| 186 | +// space by CommentGroup.Text) still matches a sentence in CONTRACT.md |
| 187 | +// that lives on a single line. |
| 188 | +func normalize(s string) string { |
| 189 | + return strings.Join(strings.Fields(s), " ") |
| 190 | +} |
| 191 | + |
| 192 | +// TestCitationsGuardCatchesFakeQuote is a regression test of the guard |
| 193 | +// itself. It builds a synthetic many-section contract and a synthetic |
| 194 | +// .go file with a real citation, a no-such-section citation, and a |
| 195 | +// quote-mismatch citation, then asserts findCitationProblems flags |
| 196 | +// exactly the two bad ones with the right diagnostics. This exists so |
| 197 | +// a future refactor of citationRE / splitContractSections / normalize |
| 198 | +// can't silently turn the guard into a no-op. |
| 199 | +func TestCitationsGuardCatchesFakeQuote(t *testing.T) { |
| 200 | + // Build a synthetic contract with at least minContractSections |
| 201 | + // `## N. <title>` sections so the floor in findCitationProblems |
| 202 | + // holds for the synthetic input. The "## 3." section is the one a |
| 203 | + // healthy citation will resolve against; the rest are filler. |
| 204 | + var contractParts []string |
| 205 | + contractParts = append(contractParts, "# Synthetic contract\n") |
| 206 | + contractParts = append(contractParts, "## 3. Date flags\n\nThe CLI accepts --since and --until.\n") |
| 207 | + for i := 1; i <= minContractSections; i++ { |
| 208 | + if i == 3 { |
| 209 | + continue |
| 210 | + } |
| 211 | + contractParts = append(contractParts, fmt.Sprintf("## %d. Filler %d\n\nFiller body.\n", i, i)) |
| 212 | + } |
| 213 | + contractMd := strings.Join(contractParts, "\n") |
| 214 | + |
| 215 | + // Synthetic source file with three citations: |
| 216 | + // - §3 quote that IS in §3 → expected to pass. |
| 217 | + // - §99 quote → expected to fail (no such section). |
| 218 | + // - §3 quote that is NOT in §3 → expected to fail (quote mismatch). |
| 219 | + // |
| 220 | + // The literal `CONTRACT §N` strings are split with concatenation |
| 221 | + // so this very file's comment scan does not flag them as real |
| 222 | + // citations of citations_test.go itself. |
| 223 | + const goSrc = `package fake |
| 224 | +
|
| 225 | +// realCite is a healthy quote that resolves to §3. |
| 226 | +// |
| 227 | +// CONTRACT ` + `§3: "accepts --since and --until" |
| 228 | +const realCite = 0 |
| 229 | +
|
| 230 | +// missingSectionCite refers to a section that does not exist. |
| 231 | +// |
| 232 | +// CONTRACT ` + `§99: "no such section" |
| 233 | +const missingSectionCite = 0 |
| 234 | +
|
| 235 | +// quoteMismatchCite refers to §3 but quotes text §3 does not contain. |
| 236 | +// |
| 237 | +// CONTRACT ` + `§3: "this exact text is fabricated" |
| 238 | +const quoteMismatchCite = 0 |
| 239 | +` |
| 240 | + tmp := t.TempDir() |
| 241 | + srcPath := filepath.Join(tmp, "fake.go") |
| 242 | + if err := os.WriteFile(srcPath, []byte(goSrc), 0o644); err != nil { |
| 243 | + t.Fatalf("write synthetic source: %v", err) |
| 244 | + } |
| 245 | + |
| 246 | + problems, err := findCitationProblems(tmp, contractMd) |
| 247 | + if err != nil { |
| 248 | + t.Fatalf("findCitationProblems: %v", err) |
| 249 | + } |
| 250 | + if got, want := len(problems), 2; got != want { |
| 251 | + t.Fatalf("got %d problems, want %d. problems=%+v\nrendered=%s", got, want, problems, formatCitationProblems(problems)) |
| 252 | + } |
| 253 | + |
| 254 | + // Problems are sorted by path then section, so §3 (the |
| 255 | + // quote-mismatch) comes before §99 (the missing-section) for the |
| 256 | + // same file. Verify each entry's section, quote, and reason shape |
| 257 | + // so a regression in any one field surfaces here. |
| 258 | + mismatch, missing := problems[0], problems[1] |
| 259 | + if missing.section != "99" || !strings.Contains(missing.reason, "no `## 99. `") { |
| 260 | + t.Errorf("expected §99 missing-section problem, got %+v", missing) |
| 261 | + } |
| 262 | + if mismatch.section != "3" || !strings.Contains(mismatch.reason, "not a substring") { |
| 263 | + t.Errorf("expected §3 quote-mismatch problem, got %+v", mismatch) |
| 264 | + } |
| 265 | + if mismatch.quote != "this exact text is fabricated" { |
| 266 | + t.Errorf("quote not captured verbatim: got %q", mismatch.quote) |
| 267 | + } |
| 268 | + |
| 269 | + // And confirm the rendered failure message names every offending |
| 270 | + // file/section/quote — that's the contract this test is locking in |
| 271 | + // for future contributors who might "simplify" the formatter. |
| 272 | + rendered := formatCitationProblems(problems) |
| 273 | + for _, want := range []string{"fake.go", "CONTRACT §3", "CONTRACT §99", "this exact text is fabricated", "no such section"} { |
| 274 | + if !strings.Contains(rendered, want) { |
| 275 | + t.Errorf("rendered failure message missing %q; got:\n%s", want, rendered) |
| 276 | + } |
| 277 | + } |
| 278 | +} |
| 279 | + |
| 280 | +// TestCitationsGuardRejectsHeaderRefactor exercises the |
| 281 | +// minContractSections floor in findCitationProblems: a contract that |
| 282 | +// no longer parses to enough top-level `## N. <title>` sections must |
| 283 | +// fail loudly rather than silently let every quote pass vacuously. |
| 284 | +func TestCitationsGuardRejectsHeaderRefactor(t *testing.T) { |
| 285 | + // Two sections only — well under minContractSections. |
| 286 | + contractMd := "# Refactored contract\n\n## 1. Foo\n\nbody\n\n## 2. Bar\n\nbody\n" |
| 287 | + tmp := t.TempDir() |
| 288 | + if err := os.WriteFile(filepath.Join(tmp, "x.go"), []byte("package x\n"), 0o644); err != nil { |
| 289 | + t.Fatalf("write: %v", err) |
| 290 | + } |
| 291 | + _, err := findCitationProblems(tmp, contractMd) |
| 292 | + if err == nil { |
| 293 | + t.Fatalf("expected error about insufficient section count, got nil") |
| 294 | + } |
| 295 | + if !strings.Contains(err.Error(), "expected at least") { |
| 296 | + t.Errorf("expected error to mention expected floor; got: %v", err) |
| 297 | + } |
| 298 | +} |
0 commit comments