diff --git a/.env.local.example b/.env.local.example index 36e1aec..bc7cbd9 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,12 +1,14 @@ -# Sentry Configuration -# Get your DSN from: https://ad-capital.sentry.io/settings/projects/json-formatter/keys/ -NEXT_PUBLIC_SENTRY_DSN=your_sentry_dsn_here -SENTRY_DSN=your_sentry_dsn_here +# Copy this file to .env.local and set only what you need. +# The app builds and runs with none of these set. +# +# All JSON processing happens in the browser; your content is never sent +# anywhere, and no analytics setting changes that. -# Sentry Auth Token for source map uploads (optional) -# Generate at: https://ad-capital.sentry.io/settings/auth-tokens/ -SENTRY_AUTH_TOKEN=your_auth_token_here +# Google Analytics 4 measurement ID, e.g. G-XXXXXXXXXX. +# Optional. Leave empty and no Google script is loaded and no consent banner is +# shown. When set, GA runs behind Consent Mode: analytics_storage starts denied, +# so no cookies are written until the visitor accepts in the banner. +NEXT_PUBLIC_GA_MEASUREMENT_ID= -# Sentry Organization and Project -SENTRY_ORG=ad-capital -SENTRY_PROJECT=json-formatter \ No newline at end of file +# Vercel Web Analytics needs no variable — it is enabled per-project from the +# Vercel dashboard, sets no cookies, and runs regardless of the consent choice. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c53d9a4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,93 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check: + name: Biome, ESLint, types & unit tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v7 + + # No `version:` input on purpose — action-setup then reads the + # `packageManager` pin from package.json, so the pin stays authoritative. + # Must run before setup-node, which needs pnpm on PATH for `cache: pnpm`. + - name: Set up pnpm + uses: pnpm/action-setup@v6 + + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version: 22.x + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run checks + run: pnpm check + + e2e: + name: Playwright smoke tests + runs-on: ubuntu-latest + # The Playwright webServer allows 300s just for the production build, on top + # of install and the browser download. + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up pnpm + uses: pnpm/action-setup@v6 + + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version: 22.x + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Cache Playwright browsers + uses: actions/cache@v6 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-playwright- + + # Safe with or without a cache hit: an already-present browser is not + # re-downloaded, and the OS-level deps are installed either way. + - name: Install Chromium + run: pnpm exec playwright install --with-deps chromium + + # No build or server step here on purpose: playwright.config.ts owns both + # via its `webServer`, which builds into .next-e2e (NEXT_DIST_DIR). + # Its CI reporter is `github`, which emits annotations but writes no report + # directory, so `html` is added on the CLI to leave something uploadable. + - name: Run Playwright tests + run: pnpm test:e2e --reporter=github,html + + - name: Upload Playwright report + if: ${{ failure() }} + uses: actions/upload-artifact@v7 + with: + name: playwright-report + path: | + playwright-report/ + test-results/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index 594ed1b..f7b6bf7 100644 --- a/.gitignore +++ b/.gitignore @@ -103,8 +103,6 @@ pip-delete-this-directory.txt # Temporary files tmp/ temp/ -# Self-hosted Monaco runtime (copied from node_modules at build time) -/public/monaco/ # Playwright /test-results/ diff --git a/LANGUAGE_TESTING.md b/LANGUAGE_TESTING.md deleted file mode 100644 index f3fb72e..0000000 --- a/LANGUAGE_TESTING.md +++ /dev/null @@ -1,85 +0,0 @@ -# Language Auto-Detection Testing Guide - -## Quick Test Methods - -### 1. Chrome DevTools Sensors (Recommended) -1. Open your site: http://localhost:3000 -2. Open DevTools (F12 or Cmd+Option+I on Mac) -3. Press **Cmd+Shift+P** (Mac) or **Ctrl+Shift+P** (Windows) -4. Type "Show Sensors" and select it -5. In the Sensors panel, find **"Locale"** dropdown -6. Select different languages: - - `zh` or `zh-CN` - Mandarin Chinese - - `es` or `es-ES` - Spanish - - `hi` or `hi-IN` - Hindi - - `tr` or `tr-TR` - Turkish - - `nl` or `nl-NL` - Dutch - - `ms` or `ms-MY` - Malay - - `ta` or `ta-IN` - Tamil -7. **Refresh the page** to trigger auto-detection - -### 2. Chrome with Language Flag (Mac/Linux) -```bash -# Mandarin Chinese -open -na "Google Chrome" --args --lang=zh --user-data-dir="/tmp/chrome-zh" http://localhost:3000 - -# Spanish -open -na "Google Chrome" --args --lang=es --user-data-dir="/tmp/chrome-es" http://localhost:3000 - -# Hindi -open -na "Google Chrome" --args --lang=hi --user-data-dir="/tmp/chrome-hi" http://localhost:3000 -``` - -### 3. Chrome Settings Change -1. Go to Chrome Settings → Advanced → Languages -2. Add Chinese (Simplified) or other languages -3. Move it to the top of the list -4. Click "Display Google Chrome in this language" -5. Restart Chrome -6. Open your site in an incognito window (to bypass localStorage) - -### 4. Browser Console Override (Temporary) -```javascript -// Run this in console before page loads -Object.defineProperty(navigator, 'language', { - get: function() { return 'zh'; } -}); -Object.defineProperty(navigator, 'languages', { - get: function() { return ['zh', 'zh-CN']; } -}); -// Then refresh the page -``` - -## Testing Checklist - -- [ ] Open site in incognito/private mode (to test fresh detection) -- [ ] Browser language set to Mandarin → Site shows in Chinese -- [ ] Browser language set to Spanish → Site shows in Spanish -- [ ] Browser language set to unsupported language → Site shows in English -- [ ] Manual language selection persists after refresh -- [ ] Language selector shows correct flag and checkmark -- [ ] All UI elements are translated properly - -## Expected Behavior - -1. **First Visit**: Site detects browser language automatically -2. **Manual Selection**: User picks language from dropdown -3. **Return Visit**: Site remembers user's selection (localStorage) -4. **Priority**: Manual selection > localStorage > Browser language > English - -## Verification Commands - -Run this to see what language the browser reports: -```javascript -console.log('Browser Language:', navigator.language); -console.log('All Languages:', navigator.languages); -console.log('Stored Language:', localStorage.getItem('i18nextLng')); -``` - -## Reset Language Selection - -To clear the saved language preference: -```javascript -localStorage.removeItem('i18nextLng'); -location.reload(); -``` \ No newline at end of file diff --git a/README.md b/README.md index e577a06..854ec32 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ JSON as an interactive node-link graph — click nodes to expand/collapse, scrol ```bash pnpm install ``` - (This also copies self-hosted editor assets and installs the Lefthook git hooks.) + (This also installs the Lefthook git hooks.) 3. Run the development server (Turbopack): ```bash @@ -138,9 +138,7 @@ Bypass once with `git commit --no-verify` if needed. json-viewer/ ├── 📁 docs/ │ └── 📁 screenshots/ # README screenshots -├── 📁 public/ # Static assets (manifest, robots.txt, images, self-hosted editor assets) -├── 📁 scripts/ -│ └── 📄 copy-monaco.mjs # Copies editor assets into public/ (no CDN egress) +├── 📁 public/ # Static assets (manifest, robots.txt, images) ├── 📁 src/ │ ├── 📁 app/ # Next.js App Router │ │ ├── 📁 (app)/ # Workbench views — one URL per view diff --git a/biome.json b/biome.json index 1c5d13d..1742989 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/2.5.7/schema.json", + "$schema": "https://biomejs.dev/schemas/2.5.10/schema.json", "vcs": { "enabled": true, "clientKind": "git", @@ -7,7 +7,7 @@ }, "files": { "ignoreUnknown": true, - "includes": ["**", "!public", "!.next", "!out", "!next-env.d.ts", "!scripts"] + "includes": ["**", "!public", "!.next", "!out", "!next-env.d.ts"] }, "formatter": { "enabled": true, diff --git a/e2e/consent.spec.ts b/e2e/consent.spec.ts new file mode 100644 index 0000000..24d017d --- /dev/null +++ b/e2e/consent.spec.ts @@ -0,0 +1,93 @@ +import { expect, test } from '@playwright/test'; + +/** + * The consent gate is only meaningful if GA genuinely writes nothing before the + * visitor accepts. That is invisible from the UI, so assert on the cookie jar + * and on the Consent Mode state the tag actually sees. + * + * These run without NEXT_PUBLIC_GA_MEASUREMENT_ID set, which is the state of the + * e2e build — so they pin the "no ID configured" contract: no Google request, no + * cookie, no banner. The consent-state assertions are written to hold either + * way, so they keep their value if the suite ever runs with an ID. + */ + +const GA_HOSTS = /googletagmanager\.com|google-analytics\.com|analytics\.google\.com/; + +test('sets no Google cookie before any choice is made', async ({ page }) => { + await page.goto('/', { waitUntil: 'networkidle' }); + const cookies = await page.context().cookies(); + expect(cookies.filter((c) => c.name.startsWith('_ga'))).toEqual([]); +}); + +test('makes no Google request when no measurement ID is configured', async ({ page }) => { + const googleRequests: string[] = []; + page.on('request', (r) => { + if (GA_HOSTS.test(r.url())) googleRequests.push(r.url()); + }); + await page.goto('/', { waitUntil: 'networkidle' }); + expect(googleRequests, googleRequests.join('\n')).toEqual([]); +}); + +test('the JSON never leaves the browser', async ({ page }) => { + const outbound: string[] = []; + page.on('request', (r) => { + const url = new URL(r.url()); + if (url.host !== new URL(page.url()).host) outbound.push(r.url()); + }); + + await page.goto('/', { waitUntil: 'networkidle' }); + const secret = 'CANARY_b7f3e1a9_do_not_transmit'; + await page.locator('.cm-content').first().fill(`{"secret": "${secret}"}`); + await page.waitForTimeout(1500); + + // Nothing carrying the document may have gone anywhere, to any host. + for (const url of outbound) { + expect(url, `request leaked document content: ${url}`).not.toContain(secret); + } + const bodies = outbound.filter((u) => GA_HOSTS.test(u)); + expect(bodies, 'no analytics request should fire while typing').toEqual([]); +}); + +/** + * The gate itself, exercised only when a measurement ID is baked into the build. + * NEXT_PUBLIC_* is inlined at build time, so this cannot be toggled per-test — + * run `NEXT_PUBLIC_GA_MEASUREMENT_ID=G-TEST123 pnpm test:e2e` to cover it. + * + * Verified manually against a G-VERIFY123 build: consent defaults to denied, no + * _ga cookie exists before the choice, and both _ga and _ga_ appear only + * after Accept. + */ +const gaConfigured = Boolean(process.env.NEXT_PUBLIC_GA_MEASUREMENT_ID); + +test.describe('with a measurement ID configured', () => { + test.skip(!gaConfigured, 'needs NEXT_PUBLIC_GA_MEASUREMENT_ID at build time'); + + test('defaults consent to denied and writes no cookie until accepted', async ({ page }) => { + await page.goto('/', { waitUntil: 'networkidle' }); + + const defaults = await page.evaluate(() => + ((window as { dataLayer?: unknown[] }).dataLayer ?? []) + .map((a) => Array.from(a as ArrayLike)) + .filter((a) => a[0] === 'consent' && a[1] === 'default') + ); + expect(defaults).toHaveLength(1); + expect(defaults[0][2]).toMatchObject({ analytics_storage: 'denied' }); + + const before = (await page.context().cookies()).filter((c) => c.name.startsWith('_ga')); + expect(before, 'no analytics cookie may exist before consent').toEqual([]); + + await page.getByRole('button', { name: /accept/i }).click(); + await page.waitForTimeout(1000); + + const after = (await page.context().cookies()).filter((c) => c.name.startsWith('_ga')); + expect(after.length, 'accepting should allow the analytics cookie').toBeGreaterThan(0); + }); + + test('declining keeps it cookieless', async ({ page }) => { + await page.goto('/', { waitUntil: 'networkidle' }); + await page.getByRole('button', { name: /decline/i }).click(); + await page.waitForTimeout(1000); + const cookies = (await page.context().cookies()).filter((c) => c.name.startsWith('_ga')); + expect(cookies, 'declining must not write an analytics cookie').toEqual([]); + }); +}); diff --git a/e2e/csp.spec.ts b/e2e/csp.spec.ts new file mode 100644 index 0000000..cd374f5 --- /dev/null +++ b/e2e/csp.spec.ts @@ -0,0 +1,78 @@ +import { expect, test } from '@playwright/test'; + +/** + * Pins the production Content-Security-Policy and proves every route renders + * without violating it. + * + * The policy was tightened once the dead Monaco stack ('unsafe-eval'), Google + * Analytics and the Google Fonts hosts were removed. Without this test those + * allowances creep back the first time something appears not to work, and a CSP + * regression is otherwise invisible: browsers report violations to the console, + * not to the test runner. + */ + +/** + * Hosts and keywords the policy must no longer permit. + * + * Google Analytics is deliberately NOT in this list: it was reinstated behind + * Consent Mode for audience metrics, so googletagmanager and the analytics + * beacon hosts are expected. What must stay gone is everything nothing uses. + */ +const REMOVED = ['unsafe-eval', 'fonts.googleapis', 'fonts.gstatic', 'openstreetmap']; + +/** Hosts the policy must still permit, or the feature that needs them breaks. */ +const REQUIRED = [ + 'https://*.basemaps.cartocdn.com', + 'https://server.arcgisonline.com', + 'https://*.googletagmanager.com', + 'https://*.google-analytics.com', +]; + +const ROUTES = ['/', '/tree', '/diff', '/graph', '/stats', '/search', '/map', '/help', '/guides']; + +/** + * Vercel serves the Web Analytics script from /_vercel/insights/* at the edge. + * Running the production build under plain `next start` there is no such route, + * so it 404s to text/plain and the browser reports a MIME error. That is an + * artefact of testing off-platform, not a policy failure — and notably NOT a CSP + * refusal, which confirms 'self' already covers the script. + */ +const OFF_PLATFORM_NOISE = /_vercel\/insights/; + +test('production CSP is the tightened policy', async ({ request }) => { + const csp = (await request.get('/')).headers()['content-security-policy']; + expect(csp, 'CSP header must be present on a production build').toBeTruthy(); + + for (const gone of REMOVED) { + expect(csp, `policy should no longer allow ${gone}`).not.toContain(gone); + } + + // Every third-party host is named rather than allowed via a blanket `https:`. + for (const host of REQUIRED) { + expect(csp, `policy must still allow ${host}`).toContain(host); + } + expect(csp).not.toMatch(/img-src[^;]*\shttps:(\s|;|$)/); + + // Clickjacking and base-tag hijacking stay closed. + expect(csp).toContain("frame-ancestors 'none'"); + expect(csp).toContain("base-uri 'self'"); + expect(csp).toContain("object-src 'none'"); +}); + +for (const path of ROUTES) { + test(`renders ${path} with no CSP violation`, async ({ page }) => { + const violations: string[] = []; + page.on('console', (msg) => { + const text = msg.text(); + if (/Content Security Policy|Refused to/i.test(text) && !OFF_PLATFORM_NOISE.test(text)) { + violations.push(`console: ${text}`); + } + }); + page.on('pageerror', (err) => { + if (!OFF_PLATFORM_NOISE.test(err.message)) violations.push(`pageerror: ${err.message}`); + }); + + await page.goto(path, { waitUntil: 'networkidle' }); + expect(violations, violations.join('\n')).toEqual([]); + }); +} diff --git a/next.config.js b/next.config.js index 42a0b3a..58501e1 100644 --- a/next.config.js +++ b/next.config.js @@ -4,22 +4,40 @@ const isProd = process.env.NODE_ENV === 'production'; // Content-Security-Policy applied in production only (dev needs 'unsafe-eval' + // websockets for HMR, which we don't want to bless permanently). +// // 'unsafe-inline' covers Next's inline bootstrap/hydration scripts and the -// JSON-LD blocks; 'unsafe-eval' is required by Monaco's AMD loader. These can be -// tightened to a nonce-based policy later via middleware. -// GA4 (gtag) loads from googletagmanager.com and posts collect beacons to -// google-analytics.com / analytics.google.com. Wildcard subdomains follow -// Google's official CSP guidance: EU traffic is routed to regional endpoints -// (e.g. region1.analytics.google.com), which the bare domains would block. -// www.google.com covers the consent-mode dual-collect fallback. +// JSON-LD blocks, and is the one remaining relaxation. Removing it needs a +// nonce, which needs middleware, which would make these statically prerendered +// pages dynamic — a deliberate trade left for its own change. +// +// Everything else is named explicitly. Deliberately absent: +// - 'unsafe-eval' only Monaco's AMD loader ever needed it. +// - Google Fonts hosts next/font/google self-hosts at build time. +// - a nonce see above; 'unsafe-inline' is the standing trade. +// - img-src https: a blanket allowance that let any host serve images; +// the two map tile providers are named instead. +// - *.tile.openstreetmap no map style has pointed there for some time. +// +// va.vercel-scripts.com is the analytics script's off-Vercel fallback; on a +// Vercel deployment the same-origin path is used, but naming it keeps analytics +// working if that ever changes. +const TILE_HOSTS = 'https://*.basemaps.cartocdn.com https://server.arcgisonline.com'; +const ANALYTICS_HOST = 'https://va.vercel-scripts.com'; +// GA4 loads from googletagmanager and beacons to google-analytics. Wildcards +// follow Google's own CSP guidance: EU traffic is routed to regional endpoints +// (region1.analytics.google.com etc.) that the bare domains would block. +const GA_SCRIPT_HOST = 'https://*.googletagmanager.com'; +const GA_BEACON_HOSTS = + 'https://*.google-analytics.com https://*.analytics.google.com https://*.googletagmanager.com'; + const contentSecurityPolicy = [ "default-src 'self'", - "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.googletagmanager.com", - "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com", - "font-src 'self' https://fonts.gstatic.com data:", - "img-src 'self' data: blob: https:", - "connect-src 'self' https://*.google-analytics.com https://*.analytics.google.com https://*.googletagmanager.com https://www.google.com https://*.tile.openstreetmap.org", - "worker-src 'self' blob: data:", + `script-src 'self' 'unsafe-inline' ${ANALYTICS_HOST} ${GA_SCRIPT_HOST}`, + "style-src 'self' 'unsafe-inline'", + "font-src 'self' data:", + `img-src 'self' data: blob: ${TILE_HOSTS} ${GA_BEACON_HOSTS}`, + `connect-src 'self' ${ANALYTICS_HOST} ${GA_BEACON_HOSTS}`, + "worker-src 'self' blob:", "frame-ancestors 'none'", "base-uri 'self'", "form-action 'self'", diff --git a/package.json b/package.json index 8ef980b..3f835a7 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,10 @@ "name": "json-viewer-react", "version": "1.0.0", "private": true, - "packageManager": "pnpm@11.20.0", + "packageManager": "pnpm@11.22.0", "scripts": { "dev": "next dev", "build": "next build", - "prebuild": "node scripts/copy-monaco.mjs", - "postinstall": "node scripts/copy-monaco.mjs", "prepare": "git rev-parse --git-dir > /dev/null 2>&1 && lefthook install || true", "start": "next start", "format": "biome format --write .", @@ -25,15 +23,14 @@ "dependencies": { "@codemirror/lang-json": "^6.0.2", "@lezer/highlight": "^1.2.3", - "@monaco-editor/react": "^4.7.0", "@uiw/codemirror-themes": "^4.25.11", "@uiw/react-codemirror": "^4.25.11", + "@vercel/analytics": "^2.0.1", "d3": "^7.9.0", "i18next": "^26.4.0", "i18next-browser-languagedetector": "^8.2.1", "leaflet": "^1.9.4", "lucide-react": "^1.33.0", - "monaco-editor": "^0.56.0", "next": "16.3.2", "notyf": "^3.10.0", "react": "^19.2.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 19a831d..dc7fb39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,15 +17,15 @@ importers: '@lezer/highlight': specifier: ^1.2.3 version: 1.2.3 - '@monaco-editor/react': - specifier: ^4.7.0 - version: 4.7.0(monaco-editor@0.56.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@uiw/codemirror-themes': specifier: ^4.25.11 version: 4.25.11(@codemirror/language@6.12.4)(@codemirror/state@6.7.1)(@codemirror/view@6.43.9) '@uiw/react-codemirror': specifier: ^4.25.11 version: 4.25.11(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.20.2)(@codemirror/language@6.12.4)(@codemirror/lint@6.9.6)(@codemirror/search@6.7.0)(@codemirror/state@6.7.1)(@codemirror/theme-one-dark@6.1.3)(@codemirror/view@6.43.9)(codemirror@6.0.2)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + '@vercel/analytics': + specifier: ^2.0.1 + version: 2.0.1(next@16.3.2(@babel/core@8.0.1)(@playwright/test@1.62.1)(@types/node@26.2.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) d3: specifier: ^7.9.0 version: 7.9.0 @@ -41,9 +41,6 @@ importers: lucide-react: specifier: ^1.33.0 version: 1.33.0(react@19.2.8) - monaco-editor: - specifier: ^0.56.0 - version: 0.56.0 next: specifier: 16.3.2 version: 16.3.2(@babel/core@8.0.1)(@playwright/test@1.62.1)(@types/node@26.2.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -717,16 +714,6 @@ packages: '@marijn/find-cluster-break@1.0.2': resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==} - '@monaco-editor/loader@1.7.0': - resolution: {integrity: sha512-gIwR1HrJrrx+vfyOhYmCZ0/JcWqG5kbfG7+d3f/C1LXk2EvzAbHSg3MQ5lO2sMlo9izoAZ04shohfKLVT6crVA==} - - '@monaco-editor/react@4.7.0': - resolution: {integrity: sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA==} - peerDependencies: - monaco-editor: '>= 0.25.0 < 1' - react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - '@next/env@16.3.2': resolution: {integrity: sha512-8k4YoG8cM7LWlkfzGNYCRBbFNlernLiMw4s0btVl+CmmWqn3VpYypA72/5Feb1UWdxe6tHqr5KHP4p4Y4m9luA==} @@ -1134,9 +1121,6 @@ packages: '@types/react@19.2.18': resolution: {integrity: sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==} - '@types/trusted-types@2.0.7': - resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} - '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} engines: {node: '>=16.20.0'} @@ -1286,6 +1270,35 @@ packages: react: '>=17.0.0' react-dom: '>=17.0.0' + '@vercel/analytics@2.0.1': + resolution: {integrity: sha512-MTQG6V9qQrt1tsDeF+2Uoo5aPjqbVPys1xvnIftXSJYG2SrwXRHnqEvVoYID7BTruDz4lCd2Z7rM1BdkUehk2g==} + peerDependencies: + '@remix-run/react': ^2 + '@sveltejs/kit': ^1 || ^2 + next: '>= 13' + nuxt: '>= 3' + react: ^18 || ^19 || ^19.0.0-rc + svelte: '>= 4' + vue: ^3 + vue-router: ^4 + peerDependenciesMeta: + '@remix-run/react': + optional: true + '@sveltejs/kit': + optional: true + next: + optional: true + nuxt: + optional: true + react: + optional: true + svelte: + optional: true + vue: + optional: true + vue-router: + optional: true + '@vitest/expect@4.1.11': resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==} @@ -1632,9 +1645,6 @@ packages: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} - dompurify@3.4.8: - resolution: {integrity: sha512-yb1cEmaOum7wFvOCSQxyfgVlv5D47Rc30iZWoMpbDIWTnJ6grDDQyu2KFJzB2k7u0pMuJcQ1zphH//fFnw2tjQ==} - dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -2337,11 +2347,6 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - marked@14.0.0: - resolution: {integrity: sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==} - engines: {node: '>= 18'} - hasBin: true - math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -2361,9 +2366,6 @@ packages: minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} - monaco-editor@0.56.0: - resolution: {integrity: sha512-sXboRm3BeBeLm938eaiyLMe0OxzfXIlZvbv4ir/jVgQy1zDhWjgmny0WoN45fuDKhCCQsYMbBJrv/A6jd8aCUg==} - ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -2664,9 +2666,6 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - state-local@1.0.7: - resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==} - std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} @@ -3537,17 +3536,6 @@ snapshots: '@marijn/find-cluster-break@1.0.2': {} - '@monaco-editor/loader@1.7.0': - dependencies: - state-local: 1.0.7 - - '@monaco-editor/react@4.7.0(monaco-editor@0.56.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': - dependencies: - '@monaco-editor/loader': 1.7.0 - monaco-editor: 0.56.0 - react: 19.2.8 - react-dom: 19.2.8(react@19.2.8) - '@next/env@16.3.2': {} '@next/eslint-plugin-next@16.3.2(eslint@10.9.0(jiti@2.7.0)(supports-color@7.2.0))': @@ -3876,9 +3864,6 @@ snapshots: dependencies: csstype: 3.2.3 - '@types/trusted-types@2.0.7': - optional: true - '@typescript/typescript-aix-ppc64@7.0.2': optional: true @@ -3972,6 +3957,11 @@ snapshots: - '@codemirror/lint' - '@codemirror/search' + '@vercel/analytics@2.0.1(next@16.3.2(@babel/core@8.0.1)(@playwright/test@1.62.1)(@types/node@26.2.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8)': + optionalDependencies: + next: 16.3.2(@babel/core@8.0.1)(@playwright/test@1.62.1)(@types/node@26.2.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + react: 19.2.8 + '@vitest/expect@4.1.11': dependencies: '@standard-schema/spec': 1.1.0 @@ -4381,10 +4371,6 @@ snapshots: dependencies: esutils: 2.0.3 - dompurify@3.4.8: - optionalDependencies: - '@types/trusted-types': 2.0.7 - dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -5143,8 +5129,6 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - marked@14.0.0: {} - math-intrinsics@1.1.0: {} merge2@1.4.1: {} @@ -5162,11 +5146,6 @@ snapshots: dependencies: brace-expansion: 1.1.18 - monaco-editor@0.56.0: - dependencies: - dompurify: 3.4.8 - marked: 14.0.0 - ms@2.1.3: {} nanoid@3.3.18: {} @@ -5530,8 +5509,6 @@ snapshots: stackback@0.0.2: {} - state-local@1.0.7: {} - std-env@4.2.0: {} stop-iteration-iterator@1.1.0: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c44b9f3..88a3133 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,19 +1,6 @@ allowBuilds: - '@sentry/cli': true - core-js: true lefthook: true sharp: true - unrs-resolver: true -minimumReleaseAgeExclude: - - '@biomejs/biome@2.5.7' - - '@biomejs/cli-darwin-arm64@2.5.7' - - '@biomejs/cli-darwin-x64@2.5.7' - - '@biomejs/cli-linux-arm64-musl@2.5.7' - - '@biomejs/cli-linux-arm64@2.5.7' - - '@biomejs/cli-linux-x64-musl@2.5.7' - - '@biomejs/cli-linux-x64@2.5.7' - - '@biomejs/cli-win32-arm64@2.5.7' - - '@biomejs/cli-win32-x64@2.5.7' patchedDependencies: eslint-plugin-react@7.37.5: patches/eslint-plugin-react@7.37.5.patch peerDependencyRules: diff --git a/public/suppress-monaco-warnings.js b/public/suppress-monaco-warnings.js deleted file mode 100644 index ead50c3..0000000 --- a/public/suppress-monaco-warnings.js +++ /dev/null @@ -1,59 +0,0 @@ -// Suppress the specific (benign) Monaco Editor messages that appear when workers -// are intentionally disabled. Scoped to exact known phrases so genuine app errors -// — including unrelated ones that merely mention "worker" — are still logged. -(function () { - if (typeof window === 'undefined') return; - - // Exact phrases emitted by Monaco when running without web workers. - var MONACO_NOISE = [ - 'Could not create web worker', - 'Falling back to loading web worker', - 'You must define a function MonacoEnvironment.getWorkerUrl', - 'FAILED to post message', - 'postMessage is not a function', - ]; - - function isMonacoNoise(args) { - var message = args && args[0] ? String(args[0]) : ''; - return MONACO_NOISE.some(function (phrase) { - return message.indexOf(phrase) !== -1; - }); - } - - var originalWarn = console.warn; - console.warn = function () { - if (isMonacoNoise(arguments)) return; - originalWarn.apply(console, arguments); - }; - - var originalError = console.error; - console.error = function () { - if (isMonacoNoise(arguments)) return; - originalError.apply(console, arguments); - }; - - // Disable Monaco web workers — runs the editor in synchronous mode (reliable, - // and avoids worker/CORS issues). Monaco assets are served same-origin. - window.MonacoEnvironment = { - getWorkerUrl: function () { - return 'data:text/javascript;charset=utf-8,'; - }, - }; - - // Swallow only the matching uncaught worker errors; let everything else through. - var originalOnError = window.onerror; - window.onerror = function (message, source, lineno, colno, error) { - if ( - typeof message === 'string' && - MONACO_NOISE.some(function (phrase) { - return message.indexOf(phrase) !== -1; - }) - ) { - return true; // Prevent default error handling for this known-benign case. - } - if (originalOnError) { - return originalOnError(message, source, lineno, colno, error); - } - return false; - }; -})(); diff --git a/scripts/copy-monaco.mjs b/scripts/copy-monaco.mjs deleted file mode 100644 index dcc10ba..0000000 --- a/scripts/copy-monaco.mjs +++ /dev/null @@ -1,21 +0,0 @@ -// Copies the Monaco Editor runtime from node_modules into public/ so it can be -// served from the same origin instead of a third-party CDN. Running locally -// keeps user data private (no request to jsdelivr/unpkg), works offline, and -// removes a supply-chain risk. Wired into `postinstall` and `prebuild`. -import { existsSync, rmSync, cpSync, mkdirSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const root = join(dirname(fileURLToPath(import.meta.url)), '..'); -const src = join(root, 'node_modules', 'monaco-editor', 'min', 'vs'); -const dest = join(root, 'public', 'monaco', 'vs'); - -if (!existsSync(src)) { - console.warn('[copy-monaco] monaco-editor not found at', src, '— skipping copy'); - process.exit(0); -} - -rmSync(dest, { recursive: true, force: true }); -mkdirSync(dirname(dest), { recursive: true }); -cpSync(src, dest, { recursive: true }); -console.log('[copy-monaco] Copied Monaco runtime → public/monaco/vs'); diff --git a/src/app/global-error.tsx b/src/app/global-error.tsx index 45e96b5..b22c348 100644 --- a/src/app/global-error.tsx +++ b/src/app/global-error.tsx @@ -14,7 +14,7 @@ export default function GlobalError({ }, [error]); return ( - +
- )} diff --git a/src/components/ClientNavbar.tsx b/src/components/ClientNavbar.tsx index abd8b8e..3540c99 100644 --- a/src/components/ClientNavbar.tsx +++ b/src/components/ClientNavbar.tsx @@ -52,7 +52,7 @@ const ClientNavbar = () => { = { added: { label: 'Added', item: 'itemAdded', badge: 'badgeAdded', pill: 'pillAdded' }, removed: { label: 'Removed', item: 'itemRemoved', badge: 'badgeRemoved', pill: 'pillRemoved' }, @@ -117,6 +136,7 @@ const CompareWorkspace: React.FC = ({ theme = 'dark', onExit, }) => { + const { t } = useTranslation(); const { showWarning, showError } = useNotification(); const [leftData, setLeftData] = useState(undefined); @@ -216,17 +236,32 @@ const CompareWorkspace: React.FC = ({ // Textual line diff over normalized (pretty-printed) JSON, so whitespace / // key-order noise never masquerades as a real change. - const blocks = useMemo(() => { - if (!ready) return []; - return computeLineDiff(normalize(leftData, sortKeys), normalize(rightData, sortKeys)); + // Both diffs recurse, and `normalize` also runs the engine's own recursive + // `JSON.stringify`, so each is guarded: a document too deep to walk shows the + // notice instead of throwing out of render into the error boundary. + const blocksResult = useMemo(() => { + if (!ready) return null; + return runDepthGuarded(() => + computeLineDiff(normalize(leftData, sortKeys), normalize(rightData, sortKeys)) + ); }, [ready, leftData, rightData, sortKeys]); + const lineDiff = blocksResult?.ok ? blocksResult.value : NO_LINE_DIFF; + const blocks = lineDiff.blocks; const stats = useMemo(() => lineStats(blocks), [blocks]); - const structural = useMemo( - () => (ready ? compareStructured(leftData, rightData) : []), + const structuralResult = useMemo( + () => (ready ? runDepthGuarded(() => compareStructured(leftData, rightData)) : null), [ready, leftData, rightData] ); + const structuralDiff = structuralResult?.ok ? structuralResult.value : NO_STRUCTURAL; + const structural = structuralDiff.diffs; + + // Non-null when either walk gave up. Without this the empty results below would + // read as "no differences found", which is the one wrong thing to say. + const depthLimit = + (blocksResult && !blocksResult.ok ? blocksResult.limit : null) ?? + (structuralResult && !structuralResult.ok ? structuralResult.limit : null); const semanticCounts = useMemo( () => ({ @@ -252,7 +287,28 @@ const CompareWorkspace: React.FC = ({ }); }, [structural, semanticFilter, query]); - const identical = ready && stats.hunks === 0 && structural.length === 0; + const identical = ready && depthLimit === null && stats.hunks === 0 && structural.length === 0; + + // Whichever cap the engine hit, say so — in the words of the view the user is + // actually reading. `lcsFallback` only describes the line diff (Split / + // Unified) and `positionalArrays` only the structural one (Semantic), so a + // single merged notice would warn about a result that is off screen. Nothing + // is claimed when the two documents came out identical: a positional walk can + // over-report edits, never invent them, so "no differences" is still exact. + let degradedNotice: string | null = null; + if (ready && depthLimit === null && !identical) { + const positionalArrays = structuralDiff.positionalArrays; + if (viewMode === 'semantic' && positionalArrays.length > 0) { + const listed = positionalArrays.slice(0, MAX_LISTED_PATHS).join(', '); + degradedNotice = t('compare.positionalArraysNotice', { + defaultValue: POSITIONAL_ARRAYS_NOTICE, + limit: MAX_ARRAY_ALIGN_ITEMS.toLocaleString(), + paths: positionalArrays.length > MAX_LISTED_PATHS ? `${listed}, …` : listed, + }); + } else if (viewMode !== 'semantic' && lineDiff.lcsFallback) { + degradedNotice = t('compare.lineFallbackNotice', { defaultValue: LINE_FALLBACK_NOTICE }); + } + } // Jump between change hunks in the scrollable results area. const navigate = useCallback((dir: 1 | -1) => { @@ -633,6 +689,13 @@ const CompareWorkspace: React.FC = ({
); } + if (depthLimit !== null) { + return ( +
+ +
+ ); + } if (identical) { return (
@@ -752,7 +815,7 @@ const CompareWorkspace: React.FC = ({
{/* Summary bar */} - {ready && !identical && ( + {ready && !identical && depthLimit === null && (
{viewMode === 'semantic' ? (
@@ -808,6 +871,17 @@ const CompareWorkspace: React.FC = ({
)} + {/* Degraded-accuracy notice: the diff below is real, but coarser than + usual, and the reader has no way to tell from the diff itself. */} + {degradedNotice && ( +
+ {degradedNotice} +
+ )} + {/* Results */}
{renderResults()} diff --git a/src/components/ConsentBanner.tsx b/src/components/ConsentBanner.tsx new file mode 100644 index 0000000..ea2fda2 --- /dev/null +++ b/src/components/ConsentBanner.tsx @@ -0,0 +1,66 @@ +'use client'; + +import { useSyncExternalStore } from 'react'; +import { useTranslation } from 'react-i18next'; +import { + type ConsentChoice, + getConsentServerSnapshot, + getConsentSnapshot, + setConsent, + subscribeToConsent, +} from '@/lib/consent'; + +/** + * Asks once whether analytics cookies are allowed. + * + * Only shown when there is a decision to make: no GA measurement ID means no + * cookies, so no banner. Until the visitor chooses, Consent Mode keeps GA + * cookieless, so declining is a real option rather than a dark pattern — both + * buttons carry equal visual weight for the same reason. + * + * Rendered from the root layout so the choice applies site-wide. + */ +export default function ConsentBanner() { + const { t } = useTranslation(); + const consent = useSyncExternalStore( + subscribeToConsent, + getConsentSnapshot, + getConsentServerSnapshot + ); + + if (!process.env.NEXT_PUBLIC_GA_MEASUREMENT_ID || consent !== 'undecided') return null; + + const decide = (choice: ConsentChoice) => setConsent(choice); + + return ( +
+
+

+ {t( + 'consent.body', + 'We use analytics cookies to count visitors and see which countries they come from. Your JSON is always processed in your browser and is never sent anywhere, whichever you choose.' + )} +

+
+ + +
+
+
+ ); +} diff --git a/src/components/DepthLimitNotice.tsx b/src/components/DepthLimitNotice.tsx new file mode 100644 index 0000000..7c6017b --- /dev/null +++ b/src/components/DepthLimitNotice.tsx @@ -0,0 +1,53 @@ +'use client'; + +import type React from 'react'; +import { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { MAX_JSON_DEPTH } from '@/utils/jsonWalk'; + +const TITLE_FALLBACK = 'Nested too deeply to display'; +const BODY_FALLBACK = + 'This document nests more than {{limit}} levels deep. Browsers run out of stack space on structures that deep, so this view cannot render it. Flatten or split the document to view it here.'; + +interface DepthLimitNoticeProps { + /** The limit that was actually hit. Defaults to the shared one. */ + limit?: number; +} + +/** + * The message every view shows when a document is nested past + * {@link MAX_JSON_DEPTH}. + * + * A shared block rather than five copies, because the explanation is the same + * everywhere and it is the one thing standing between the user and a bare + * "Maximum call stack size exceeded" from the error boundary. Each view wraps it + * in its own container, so it slots into whatever empty state that view already + * uses. + * + * The `mounted` gate matches the rest of the app: the English fallback renders + * on the server and during the first client render, and the translated string + * appears once i18next has resolved the browser's language, so the two markups + * agree. + */ +const DepthLimitNotice: React.FC = ({ limit = MAX_JSON_DEPTH }) => { + const { t } = useTranslation(); + const [mounted, setMounted] = useState(false); + + useEffect(() => { + setMounted(true); + }, []); + + const title = mounted ? t('depth.title', { defaultValue: TITLE_FALLBACK }) : TITLE_FALLBACK; + const body = mounted + ? t('depth.body', { defaultValue: BODY_FALLBACK, limit }) + : BODY_FALLBACK.replace('{{limit}}', String(limit)); + + return ( +
+

{title}

+

{body}

+
+ ); +}; + +export default DepthLimitNotice; diff --git a/src/components/EditorView.tsx b/src/components/EditorView.tsx index 91454d0..6e34d37 100644 --- a/src/components/EditorView.tsx +++ b/src/components/EditorView.tsx @@ -2,12 +2,10 @@ import dynamic from 'next/dynamic'; import type React from 'react'; -import { useEffect, useState } from 'react'; -import { useNotification } from '@/hooks/useNotification'; +import { describeDocumentSize } from '@/utils/jsonDocument'; // Single editor across all devices: CodeMirror 6. It's fully bundled (no CDN -// egress, matching the privacy-first stance), touch-friendly, and theme-aware — -// replacing the previous Monaco-on-desktop / CodeMirror-on-mobile split. +// egress, matching the privacy-first stance), touch-friendly, and theme-aware. const CodeMirrorEditor = dynamic(() => import('./CodeMirrorEditor'), { ssr: false, loading: () => ( @@ -23,35 +21,27 @@ interface EditorViewProps { theme: 'light' | 'dark'; } -// File size limits -const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB -const LARGE_FILE_SIZE = 2 * 1024 * 1024; // 2MB - const EditorView: React.FC = ({ content, onChange, theme }) => { - const { showError } = useNotification(); - const [isLargeFile, setIsLargeFile] = useState(false); - - // Check file size. Uses string length as a fast proxy for byte size. - useEffect(() => { - const contentSize = content.length; - if (contentSize > MAX_FILE_SIZE) { - showError( - `File too large (${(contentSize / 1024 / 1024).toFixed(2)}MB). Maximum allowed is 10MB.` - ); - onChange('{"error": "File too large. Please use a file smaller than 10MB"}'); - return; - } - setIsLargeFile(contentSize > LARGE_FILE_SIZE); - }, [content, onChange, showError]); + // Derived during render, not written from an effect. + // + // This used to be an effect that, past the 10MB cap, called + // `onChange('{"error": "File too large…"}')` — replacing whatever the user had + // just pasted with an error object. A warning must never destroy the + // document: the size cap is enforced where content enters from outside (file + // open, drag-and-drop, the Paste button), and content typed or pasted + // directly into the editor is kept and flagged here instead. + const size = describeDocumentSize(content.length); return ( -
- {isLargeFile && ( -
- Large file detected. Some features may be slower. +
+ {size !== 'normal' && ( +
+ {size === 'over-max' + ? `Very large document (${(content.length / 1024 / 1024).toFixed(2)}MB, over the 10MB limit). Nothing was discarded — editing and the other views may be slow.` + : 'Large file detected. Some features may be slower.'}
)} -
+
diff --git a/src/components/GraphView.tsx b/src/components/GraphView.tsx index c42f09b..bad19fd 100644 --- a/src/components/GraphView.tsx +++ b/src/components/GraphView.tsx @@ -5,19 +5,20 @@ import { ChevronsDownUp, ChevronsUpDown, Maximize2, ZoomIn, ZoomOut } from 'luci import type React from 'react'; import { useEffect, useRef } from 'react'; import { useTranslation } from 'react-i18next'; +import { buildTooltipModel, buildTree, type JsonNode, previewValue } from '@/utils/graphData'; +import { exceedsJsonDepth, MAX_JSON_DEPTH, runDepthGuarded } from '@/utils/jsonWalk'; +import DepthLimitNotice from './DepthLimitNotice'; import styles from './GraphView.module.css'; interface GraphViewProps { json: any; -} - -interface JsonNode { - name: string; - type: 'object' | 'array' | 'value'; - path: string; - value?: any; - valueType?: string; - children?: JsonNode[]; + /** + * Whether the editor holds a document at all. Carried explicitly rather than + * inferred from `json`, because `null`, `0`, `false` and `""` are valid JSON + * documents that a truthiness test reports as "nothing to visualise" — while + * the status bar, which does carry the flag, calls the same document valid. + */ + isValid: boolean; } // Theme-aware value color (the CSS vars resolve per light/dark automatically). @@ -36,45 +37,11 @@ function valueColor(valueType?: string): string { } } -function preview(value: any): string { - if (value === null) return 'null'; - if (typeof value === 'string') { - return value.length > 28 ? `"${value.slice(0, 28)}…"` : `"${value}"`; - } - return String(value); -} - -function buildTree(data: any, name: string, path: string): JsonNode { - if (data !== null && Array.isArray(data)) { - return { - name, - type: 'array', - path, - children: data.map((v, i) => buildTree(v, String(i), `${path}[${i}]`)), - }; - } - if (data !== null && typeof data === 'object') { - return { - name, - type: 'object', - path, - children: Object.entries(data).map(([k, v]) => buildTree(v, k, `${path}.${k}`)), - }; - } - return { - name, - type: 'value', - path, - value: data, - valueType: data === null ? 'null' : typeof data, - }; -} - const NODE_HEIGHT = 26; // vertical spacing between siblings const LEVEL_WIDTH = 250; // horizontal spacing between depths const INITIAL_DEPTH = 2; // collapse nodes at this depth and deeper on load -const GraphView: React.FC = ({ json }) => { +const GraphView: React.FC = ({ json, isValid }) => { const { t } = useTranslation(); const containerRef = useRef(null); const apiRef = useRef<{ @@ -84,17 +51,38 @@ const GraphView: React.FC = ({ json }) => { collapseAll: () => void; } | null>(null); + // Asked with the iterative probe, before anything recursive touches the + // document. Derived during render (the React Compiler memoizes it on `json`) + // rather than set from the effect, which would mean a cascading render. + const tooDeep = json != null && exceedsJsonDepth(json); + useEffect(() => { const container = containerRef.current; - if (!container || !json) return; - container.innerHTML = ''; + if (!container || !isValid || tooDeep) return; + container.replaceChildren(); + + // Every `setTimeout` scheduled by this effect lands here so cleanup can + // cancel it; a fit() that fires after unmount would call getBBox() on a + // detached , which throws NS_ERROR_FAILURE in Firefox. + const timers = new Set>(); + const scheduleFit = (delay: number) => { + const id = setTimeout(() => { + timers.delete(id); + fit(); + }, delay); + timers.add(id); + }; const rootLabel = Array.isArray(json) ? '[ ]' : json !== null && typeof json === 'object' ? '{ }' : 'value'; - const rootData = buildTree(json, rootLabel, '$'); + // Belt and braces behind the probe above: if the two ever disagreed, an + // empty canvas beats an error boundary. + const built = runDepthGuarded(() => buildTree(json, rootLabel, '$')); + if (!built.ok) return; + const rootData = built.value; const svg = d3 .select(container) @@ -152,7 +140,10 @@ const GraphView: React.FC = ({ json }) => { if (data.type === 'value') { sel.append('tspan').style('fill', 'var(--json-key)').text(data.name); sel.append('tspan').style('fill', 'var(--text-muted)').text(': '); - sel.append('tspan').style('fill', valueColor(data.valueType)).text(preview(data.value)); + sel + .append('tspan') + .style('fill', valueColor(data.valueType)) + .text(previewValue(data.value)); } else { const n = data.children ? data.children.length : 0; const wrap = data.type === 'array' ? ['[', ']'] : ['{', '}']; @@ -172,15 +163,26 @@ const GraphView: React.FC = ({ json }) => { } function showTip(event: MouseEvent, d: any) { - const data: JsonNode = d.data; - const count = data.children ? data.children.length : 0; - let html = `
${data.path}
`; - if (data.type === 'value') { - html += `
${preview(data.value)} ${data.valueType}
`; + const tip = buildTooltipModel(d.data as JsonNode); + // Built with DOM nodes and .text(), never .html(): the path and the value + // are raw user JSON, so anything that parsed them as markup would execute + // a key like `` on hover. + tooltip.selectAll('*').remove(); + tooltip.append('div').attr('class', styles.tipPath).text(tip.path); + if (tip.kind === 'value') { + // .text() wipes children, so it has to run before the type span is + // appended — the trailing space is the separator the old markup had. + tooltip + .append('div') + .attr('class', styles.tipVal) + .text(`${tip.value} `) + .append('span') + .attr('class', styles.tipType) + .text(tip.valueType); } else { - html += `
${data.type} · ${count} ${data.type === 'array' ? 'items' : 'keys'}
`; + tooltip.append('div').attr('class', styles.tipType).text(tip.summary); } - tooltip.html(html).style('opacity', '1'); + tooltip.style('opacity', '1'); moveTip(event); } function moveTip(event: MouseEvent) { @@ -280,7 +282,9 @@ const GraphView: React.FC = ({ json }) => { function fit() { const node = g.node() as SVGGElement | null; - if (!node) return; + // getBBox() on a detached / non-rendered element throws NS_ERROR_FAILURE in + // Firefox, so bail out if this graph was torn down since fit() was queued. + if (!node || !node.isConnected) return; const b = node.getBBox(); if (!b.width || !b.height) return; const fw = container!.clientWidth || 800; @@ -309,7 +313,7 @@ const GraphView: React.FC = ({ json }) => { if (d._children) d.children = d._children; }); update(root); - setTimeout(fit, 260); + scheduleFit(260); }, collapseAll: () => { root.descendants().forEach((d: any) => { @@ -319,20 +323,30 @@ const GraphView: React.FC = ({ json }) => { } }); update(root); - setTimeout(fit, 260); + scheduleFit(260); }, }; update(root); - setTimeout(fit, 60); + scheduleFit(60); return () => { + for (const id of timers) clearTimeout(id); + timers.clear(); + // The 240–300ms node/link/zoom transitions keep ticking after the nodes + // are detached, so stop them before tearing the DOM down. + svg.interrupt(); + svg.selectAll('*').interrupt(); + // d3-zoom's listeners live on the we are about to drop, but a + // gesture in flight also holds window-level ones; detaching them here + // keeps nothing pointed at this effect's closure. + svg.on('.zoom', null); apiRef.current = null; - container.innerHTML = ''; + container.replaceChildren(); }; - }, [json]); + }, [json, isValid, tooDeep]); - if (!json) { + if (!isValid) { return (

@@ -347,55 +361,71 @@ const GraphView: React.FC = ({ json }) => { const btnCls = 'w-8 h-8 flex items-center justify-center rounded-md border border-[var(--border-color)] bg-[var(--bg-secondary)] text-[var(--text-secondary)] hover:text-[var(--accent-color)] hover:border-[var(--border-hover)] transition-colors'; + const zoomInLabel = t('graph.zoomIn', 'Zoom in'); + const zoomOutLabel = t('graph.zoomOut', 'Zoom out'); + const fitLabel = t('graph.fitToScreen', 'Fit to screen'); + const expandLabel = t('graph.expandAll', 'Expand all'); + const collapseLabel = t('graph.collapseAll', 'Collapse all'); + return (

-
- - - - - -
-
- click a node to expand / collapse · scroll to zoom · drag to pan -
+ {tooDeep ? ( +
+ +
+ ) : ( + <> +
+ + + + + +
+
+ {t('graph.hint', 'click a node to expand / collapse · scroll to zoom · drag to pan')} +
+ + )} + {/* Always mounted, in both branches: unmounting it would null out + `containerRef` and the effect would then refuse every later document. */}
); diff --git a/src/components/I18nProvider-BrowserFirst.tsx b/src/components/I18nProvider-BrowserFirst.tsx deleted file mode 100644 index 2c072bf..0000000 --- a/src/components/I18nProvider-BrowserFirst.tsx +++ /dev/null @@ -1,27 +0,0 @@ -'use client'; - -import type React from 'react'; -import { useEffect } from 'react'; -import { I18nextProvider } from 'react-i18next'; -import i18n from '@/lib/i18n'; - -// This version ALWAYS prioritizes browser language over saved preference -const I18nProviderBrowserFirst: React.FC<{ children: React.ReactNode }> = ({ children }) => { - useEffect(() => { - // ALWAYS detect from browser first - const browserLang = navigator.language || navigator.languages?.[0] || 'en'; - const langCode = browserLang.split('-')[0]; - - const supportedLangs = ['en', 'es', 'hi', 'tr', 'nl', 'ms', 'zh', 'ta', 'fy', 'nds', 'li']; - const languageToUse = supportedLangs.includes(langCode) ? langCode : 'en'; - - if (languageToUse !== i18n.language) { - i18n.changeLanguage(languageToUse); - // Don't save to localStorage - always use browser preference - } - }, []); - - return {children}; -}; - -export default I18nProviderBrowserFirst; diff --git a/src/components/I18nProvider.tsx b/src/components/I18nProvider.tsx index 3cc2844..83ee961 100644 --- a/src/components/I18nProvider.tsx +++ b/src/components/I18nProvider.tsx @@ -1,7 +1,6 @@ 'use client'; import type React from 'react'; -import { useEffect } from 'react'; import { I18nextProvider } from 'react-i18next'; import i18n from '@/lib/i18n'; @@ -9,32 +8,10 @@ interface I18nProviderProps { children: React.ReactNode; } +// Language detection lives entirely in `@/lib/i18n` (LanguageDetector with +// order: ['localStorage', 'navigator', 'htmlTag']), which runs at module init. +// Don't re-implement it here — a second, divergent detector is how 'zh-CN' got lost. const I18nProvider: React.FC = ({ children }) => { - useEffect(() => { - // Check if there's a saved language preference - const savedLanguage = localStorage.getItem('i18nextLng'); - - if (savedLanguage) { - // Use saved language if it exists - if (savedLanguage !== i18n.language) { - i18n.changeLanguage(savedLanguage); - } - } else { - // No saved language, detect from browser - const browserLang = navigator.language || navigator.languages?.[0] || 'en'; - // Extract the language code (e.g., 'zh' from 'zh-CN') - const langCode = browserLang.split('-')[0]; - - // Check if we support this language - const supportedLangs = ['en', 'es', 'hi', 'tr', 'nl', 'ms', 'zh', 'ta', 'fy', 'nds', 'li']; - const languageToUse = supportedLangs.includes(langCode) ? langCode : 'en'; - - if (languageToUse !== i18n.language) { - i18n.changeLanguage(languageToUse); - } - } - }, []); - return {children}; }; diff --git a/src/components/JsonWorkbench.tsx b/src/components/JsonWorkbench.tsx index 42dcd12..1b1cb4d 100644 --- a/src/components/JsonWorkbench.tsx +++ b/src/components/JsonWorkbench.tsx @@ -15,10 +15,17 @@ import TabsContainer from '@/components/TabsContainer'; import TreeView from '@/components/TreeView'; import { useNotification } from '@/hooks/useNotification'; import { useTheme } from '@/hooks/useTheme'; -import * as gtag from '@/lib/gtag'; import { pathToView, type ToolView, viewToPath } from '@/lib/tools'; +import { + isParseSuccess, + MAX_DOCUMENT_CHARS, + parseJsonStrict, + sortObjectKeysDeep, +} from '@/utils/jsonDocument'; import { JSONFixer } from '@/utils/jsonFixer'; +import { describeLossyScan, scanLossyRewrites } from '@/utils/jsonLossy'; import { formatJSON } from '@/utils/jsonUtils'; +import { runDepthGuarded } from '@/utils/jsonWalk'; // Lazy load heavy components for better performance. Only the active view's // chunk loads, so each route pulls in just the view it opens (e.g. /graph loads @@ -50,6 +57,23 @@ const SearchView = dynamic(() => import('@/components/SearchView'), { ssr: false, }); +/** + * The parsed document. `isValid` is carried explicitly instead of being inferred + * from `data`, because `null` is both "nothing parsed" and a perfectly valid + * JSON document — as are `0`, `false` and `""`. + */ +interface DocumentState { + isValid: boolean; + data: any; +} + +const EMPTY_DOCUMENT: DocumentState = { isValid: false, data: null }; + +const SORT_DEPTH_FALLBACK = + 'This document is nested too deeply to sort its keys (limit: {{limit}} levels).'; + +const LOSSY_WARNING_FALLBACK = 'Your JSON was rewritten and some values changed: {{details}}.'; + /** * The JSON workbench. It lives in the persistent `(app)` layout, so its state * (editor content, options) survives navigation between view routes. The active @@ -86,7 +110,8 @@ export default function JsonWorkbench() { // Held here, in the persistent layout, so they survive switching between views. const [compareMode, setCompareMode] = useState(false); const [compareContent, setCompareContent] = useState(''); - const [parsedJson, setParsedJson] = useState(null); + const [parsed, setParsed] = useState(EMPTY_DOCUMENT); + const parsedJson = parsed.data; const [showExampleModal, setShowExampleModal] = useState(false); const [isUpdatingFromTree, setIsUpdatingFromTree] = useState(false); const [skipValidation, setSkipValidation] = useState(false); @@ -114,7 +139,6 @@ export default function JsonWorkbench() { // Navigate to a view's clean URL. The layout persists, so the editor content // is preserved across the navigation. const goToView = (view: ToolView) => { - gtag.trackTabSwitch(view); startTransition(() => { setOptimisticView(view); router.push(viewToPath(view), { scroll: false }); @@ -124,21 +148,51 @@ export default function JsonWorkbench() { // Indentation passed to JSON.stringify: a number of spaces, or a literal tab. const indentValue: string | number = indent === 'tab' ? '\t' : parseInt(indent, 10); - // Recursively sort object keys A→Z (arrays keep their order). - const sortObjectKeys = (value: any): any => { - if (Array.isArray(value)) return value.map(sortObjectKeys); - if (value && typeof value === 'object') { - return Object.keys(value) - .sort() - .reduce((acc: Record, key) => { - acc[key] = sortObjectKeys(value[key]); - return acc; - }, {}); - } - return value; + // Recursively sort object keys A→Z (arrays keep their order). Lives in + // src/utils/jsonDocument.ts so it can be unit-tested — the inline version it + // replaces silently dropped a member literally named `__proto__`. + // + // The sort recurses, so a document nested past the display limit cannot be + // sorted. It is still perfectly formattable, so the toast explains what was + // skipped and the unsorted document goes through rather than the click + // throwing out of its handler, where no error boundary is watching. + const sortDocument = (data: any) => { + const sorted = runDepthGuarded(() => sortObjectKeysDeep(data)); + if (sorted.ok) return sorted.value; + showError( + mounted + ? t('depth.sortFailed', { defaultValue: SORT_DEPTH_FALLBACK, limit: sorted.limit }) + : SORT_DEPTH_FALLBACK.replace('{{limit}}', String(sorted.limit)) + ); + return data; }; - const applyOptions = (data: any) => (sortKeys ? sortObjectKeys(data) : data); + const applyOptions = (data: any) => (sortKeys ? sortDocument(data) : data); + + // What Format / Compact are about to change about the document — or null when + // the round trip is faithful. Read from the text still in the editor, before + // anything overwrites it. + const describeLossyRewrite = (source: string): string | null => + describeLossyScan(scanLossyRewrites(source), (key, fallback) => + mounted ? t(key, { defaultValue: fallback }) : fallback + ); + + // Format and Compact are `JSON.parse` followed by `JSON.stringify`, and that + // round trip silently rewrites integers past `Number.MAX_SAFE_INTEGER`, numbers + // too large to represent, minus zero, and duplicate keys — all reported to the + // user today as a successful format. A genuinely lossless formatter needs its + // own number type and serialiser; saying what changed costs one scan, and is + // the difference between a surprise and a decision. + const warnAboutLossyRewrite = (details: string | null) => { + if (details === null) return; + setTimeout(() => { + showWarning( + mounted + ? t('format.lossy.warning', { defaultValue: LOSSY_WARNING_FALLBACK, details }) + : LOSSY_WARNING_FALLBACK.replace('{{details}}', details) + ); + }, 100); + }; // Re-format the current document immediately when indentation or key-sort // options change, so toggling 2 / 4 / Tab (or Sort keys) reflows the JSON @@ -146,12 +200,18 @@ export default function JsonWorkbench() { const reformatWith = (indentSel: string, sort: boolean) => { if (!editorContent || editorContent.trim() === '') return; const result = JSONFixer.parseWithFixInfo(editorContent); - if (!result.data) return; + if (!isParseSuccess(result)) return; const iv: string | number = indentSel === 'tab' ? '\t' : parseInt(indentSel, 10); - const data = sort ? sortObjectKeys(result.data) : result.data; + // Same destructive round trip as Format, reached from a control the user + // does not think of as rewriting anything. Warning here does not become + // noise: `JSON.stringify` output always scans clean, so a document that has + // already been through this fires nothing on the next toggle. + const lossy = describeLossyRewrite(editorContent); + const data = sort ? sortDocument(result.data) : result.data; setSkipValidation(true); setEditorContent(JSON.stringify(data, null, iv)); - setParsedJson(result.data); + setParsed({ isValid: true, data }); + warnAboutLossyRewrite(lossy); }; const handleIndentChange = (value: string) => { @@ -166,20 +226,29 @@ export default function JsonWorkbench() { }; useEffect(() => { - // Format/compact/paste set parsedJson directly, so skip re-parsing once. + // Format/compact/paste/open set `parsed` directly, so skip re-parsing once. + // Note this defers rather than cancels: clearing the flag re-runs the effect + // and the debounced parse still lands. That is safe only because every path + // that sets `parsed` from repaired data also writes `JSON.stringify` of that + // same data back into the editor, so the strict re-parse converges on the + // identical value. Do not add a path that sets `parsed` without doing so. if (skipValidation) { setSkipValidation(false); return; } if (isUpdatingFromTree) return; if (!editorContent) { - setParsedJson(null); + setParsed(EMPTY_DOCUMENT); return; } // Debounce parsing so large documents don't re-parse on every keystroke. + // This is a plain JSON.parse, never the repair pipeline: repair is + // super-linear on large invalid input (380KB → ~800ms) and a document is + // invalid for exactly as long as the user is still typing it. Whether the + // document is *fixable* is decided only on an explicit action below. const timeoutId = setTimeout(() => { - const result = JSONFixer.parseWithFixInfo(editorContent); - setParsedJson(result.error ? null : result.data); + const result = parseJsonStrict(editorContent); + setParsed({ isValid: result.isValid, data: result.data }); }, 250); return () => clearTimeout(timeoutId); }, [editorContent, isUpdatingFromTree, skipValidation]); @@ -191,18 +260,21 @@ export default function JsonWorkbench() { } // Track format action - gtag.trackJsonFormat(); - // Try to parse and potentially fix the JSON + // Format is the explicit action, so this is where the repair pipeline runs. const result = JSONFixer.parseWithFixInfo(editorContent); - if (result.data) { - const formatted = JSON.stringify(applyOptions(result.data), null, indentValue); + if (isParseSuccess(result)) { + const lossy = describeLossyRewrite(editorContent); + const data = applyOptions(result.data); + const formatted = JSON.stringify(data, null, indentValue); // Set skip validation flag to prevent useEffect from running setSkipValidation(true); setEditorContent(formatted); - setParsedJson(result.data); + setParsed({ isValid: true, data }); + + warnAboutLossyRewrite(lossy); if (result.wasFixed && result.fixes && result.fixes.length > 0) { setTimeout(() => { @@ -221,18 +293,21 @@ export default function JsonWorkbench() { } // Track compact action - gtag.trackJsonCompact(); - // Try to parse and potentially fix the JSON + // Compact rewrites the document, so the repair pipeline runs here too. const result = JSONFixer.parseWithFixInfo(editorContent); - if (result.data) { - const compacted = JSON.stringify(applyOptions(result.data)); + if (isParseSuccess(result)) { + const lossy = describeLossyRewrite(editorContent); + const data = applyOptions(result.data); + const compacted = JSON.stringify(data); // Set skip validation flag to prevent useEffect from running setSkipValidation(true); setEditorContent(compacted); - setParsedJson(result.data); + setParsed({ isValid: true, data }); + + warnAboutLossyRewrite(lossy); if (result.wasFixed && result.fixes && result.fixes.length > 0) { setTimeout(() => { @@ -245,9 +320,8 @@ export default function JsonWorkbench() { }; const handleClear = () => { - gtag.trackJsonClear(); setEditorContent(''); - setParsedJson(null); + setParsed(EMPTY_DOCUMENT); }; const handleCopy = async () => { @@ -256,8 +330,6 @@ export default function JsonWorkbench() { return; } - gtag.trackJsonCopy(); - try { await navigator.clipboard.writeText(editorContent); @@ -280,8 +352,6 @@ export default function JsonWorkbench() { }; const handlePaste = async () => { - gtag.trackJsonPaste(); - try { const text = await navigator.clipboard.readText(); if (!text || text.trim() === '') { @@ -289,10 +359,22 @@ export default function JsonWorkbench() { return; } - // Try to parse and potentially fix the JSON + // Enforce the size cap at the boundary, refusing the paste rather than + // accepting it and then destroying the editor's contents. + if (text.length > MAX_DOCUMENT_CHARS) { + showError( + `Clipboard content is too large (${(text.length / 1024 / 1024).toFixed(2)}MB). Maximum allowed is 10MB.` + ); + return; + } + + // Paste is an explicit action, so the repair pipeline runs here. const result = JSONFixer.parseWithFixInfo(text); - if (result.data) { + if (isParseSuccess(result)) { + // Same round trip as Format, on text the user never had a chance to + // read, so the same warning applies. + const lossy = describeLossyRewrite(text); // JSON was successfully parsed (possibly after fixing) const formatted = JSON.stringify(result.data, null, 2); @@ -303,11 +385,13 @@ export default function JsonWorkbench() { setEditorContent(formatted); // Update parsed JSON directly - setParsedJson(result.data); + setParsed({ isValid: true, data: result.data }); // Show the editor so the pasted content is visible. router.push('/', { scroll: false }); + warnAboutLossyRewrite(lossy); + if (result.wasFixed && result.fixes && result.fixes.length > 0) { // JSON was fixed - show warning after a brief delay to ensure UI updates setTimeout(() => { @@ -342,7 +426,6 @@ export default function JsonWorkbench() { : exampleContent.includes('coordinates') ? 'geographic' : 'unknown'; - gtag.trackExampleUsed(exampleName); try { // Format the example content immediately @@ -380,15 +463,18 @@ export default function JsonWorkbench() { // Shared loader for both the file picker and drag-and-drop. const loadFromText = (text: string) => { - if (text.length > 10 * 1024 * 1024) { + if (text.length > MAX_DOCUMENT_CHARS) { showError('File too large. Maximum allowed is 10MB.'); return; } const result = JSONFixer.parseWithFixInfo(text); - if (result.data) { + if (isParseSuccess(result)) { + const lossy = describeLossyRewrite(text); + const data = applyOptions(result.data); setSkipValidation(true); - setEditorContent(JSON.stringify(applyOptions(result.data), null, indentValue)); - setParsedJson(result.data); + setEditorContent(JSON.stringify(data, null, indentValue)); + setParsed({ isValid: true, data }); + warnAboutLossyRewrite(lossy); if (result.wasFixed && result.fixes && result.fixes.length > 0) { setTimeout( () => showWarning(`JSON was automatically fixed: ${result.fixes!.join(', ')}`), @@ -521,34 +607,45 @@ export default function JsonWorkbench() { > {activeTab === 'tree' && ( - + )} {activeTab === 'graph' && ( - + )} {activeTab === 'stats' && ( - + )} {activeTab === 'map' && ( - + )} {activeTab === 'search' && ( - + )}
)}
- + {/* The status bar is the one piece of the workbench that renders + outside a view boundary, so a throw here used to unmount the whole + app and take the user's editor content with it. The fallback is an + empty strip rather than the boundary's default full-screen panel: + losing the byte count is a footnote, losing the editor is the bug. */} + }> + + {showExampleModal && ( diff --git a/src/components/LanguageSelector.tsx b/src/components/LanguageSelector.tsx index c2af675..952b9e4 100644 --- a/src/components/LanguageSelector.tsx +++ b/src/components/LanguageSelector.tsx @@ -4,7 +4,8 @@ import { Check, ChevronDown, Globe } from 'lucide-react'; import type React from 'react'; import { useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { languages } from '@/lib/i18n'; +import { canonicalLanguage, languages } from '@/lib/i18n'; +import { writeStored } from '@/utils/safeStorage'; interface LanguageSelectorProps { showLabel?: boolean; @@ -16,7 +17,12 @@ const LanguageSelector: React.FC = ({ showLabel = false } const [mounted, setMounted] = useState(false); const dropdownRef = useRef(null); - const currentLanguage = languages.find((lang) => lang.code === i18n.language) || languages[0]; + // `resolvedLanguage` is the bundle actually in use, which is what the checkmark should + // track; `i18n.language` can keep a raw region tag when `changeLanguage` is called with + // one directly. `canonicalLanguage` then folds script aliases (e.g. 'zh-Hans') onto the + // code the `languages` list actually holds, so the checkmark never falls back to English. + const activeLanguage = canonicalLanguage(i18n.resolvedLanguage || i18n.language); + const currentLanguage = languages.find((lang) => lang.code === activeLanguage) || languages[0]; useEffect(() => { setMounted(true); @@ -35,7 +41,7 @@ const LanguageSelector: React.FC = ({ showLabel = false } const handleLanguageChange = (languageCode: string) => { i18n.changeLanguage(languageCode); - localStorage.setItem('i18nextLng', languageCode); + writeStored('i18nextLng', languageCode); setIsOpen(false); }; diff --git a/src/components/MapView.tsx b/src/components/MapView.tsx index 3a1a7bd..d10aa9f 100644 --- a/src/components/MapView.tsx +++ b/src/components/MapView.tsx @@ -24,9 +24,10 @@ const DynamicMapComponent = dynamic( interface MapViewProps { json: any; + isValid?: boolean; } -const MapView: React.FC = ({ json }) => { +const MapView: React.FC = ({ json, isValid }) => { const { t } = useTranslation(); const [mounted, setMounted] = useState(false); @@ -34,7 +35,7 @@ const MapView: React.FC = ({ json }) => { setMounted(true); }, []); - return ; + return ; }; export default MapView; diff --git a/src/components/MapViewDynamic.tsx b/src/components/MapViewDynamic.tsx index 9d258a0..4373e7c 100644 --- a/src/components/MapViewDynamic.tsx +++ b/src/components/MapViewDynamic.tsx @@ -1,278 +1,360 @@ 'use client'; import L from 'leaflet'; -import dynamic from 'next/dynamic'; import type React from 'react'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import { GeoJSON, MapContainer, Marker, Popup, TileLayer, useMap } from 'react-leaflet'; +import { useThemeContext } from '@/components/ThemeProvider'; +import { + EMPTY_EXTRACTION, + extractGeoData, + type GeoLocation, + MAP_STYLES, + type MapStyleId, + serializeTileConsent, + shapesSignature, + TILE_CONSENT_KEY, + TILE_PROVIDERS, + tileConsentCovers, + toDisplayString, + toFeatureCollection, + viewFromBounds, +} from '@/utils/geoDetect'; +import { runDepthGuarded } from '@/utils/jsonWalk'; +import DepthLimitNotice from './DepthLimitNotice'; import 'leaflet/dist/leaflet.css'; - -// Fix for default markers not showing -delete (L.Icon.Default.prototype as any)._getIconUrl; -L.Icon.Default.mergeOptions({ - iconRetinaUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.7.1/images/marker-icon-2x.png', - iconUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.7.1/images/marker-icon.png', - shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.7.1/images/marker-shadow.png', -}); - -const MapContainer = dynamic(() => import('react-leaflet').then((mod) => mod.MapContainer), { - ssr: false, -}); - -const TileLayer = dynamic(() => import('react-leaflet').then((mod) => mod.TileLayer), { - ssr: false, -}); - -const Marker = dynamic(() => import('react-leaflet').then((mod) => mod.Marker), { ssr: false }); - -const Popup = dynamic(() => import('react-leaflet').then((mod) => mod.Popup), { ssr: false }); - -const GeoJSON = dynamic(() => import('react-leaflet').then((mod) => mod.GeoJSON), { ssr: false }); +import { readStored, writeStored } from '@/utils/safeStorage'; interface MapViewProps { json: any; + /** + * Whether `json` is a parsed document at all. Carried explicitly because `null`, `0`, + * `false` and `""` are valid JSON documents that a truthiness test reads as "nothing + * loaded". Optional only until `MapView` forwards it; the fallback still distinguishes + * every falsy document except `null`. + */ + isValid?: boolean; } -interface LocationData { - lat: number; - lng: number; - data: any; - path: string; - name?: string; - properties?: any; +/** + * Tile requests are z/x/y derived from the coordinates in the user's private document, sent to a + * third party along with their IP. The app promises the JSON never leaves the browser, so tiles + * stay off until the user asks for them; the choice is remembered per browser. + * + * The grant is stored as the list of hosts disclosed at the time (see `tileConsentCovers`), so a + * provider added later cannot inherit an agreement that never named it. + */ +function readTileConsent(): boolean { + try { + return tileConsentCovers(readStored(TILE_CONSENT_KEY)); + } catch { + return false; + } } -type MapStyle = 'dark' | 'light' | 'satellite'; +function storeTileConsent(): void { + try { + writeStored(TILE_CONSENT_KEY, serializeTileConsent()); + } catch { + // Private mode or blocked storage — consent then lasts for this session only. + } +} -const mapStyles: Record = { - dark: { - name: 'Dark', - url: 'https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', - attribution: - '© OpenStreetMap contributors © CARTO', - }, - light: { - name: 'Light', - url: 'https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', - attribution: - '© OpenStreetMap contributors © CARTO', - }, - satellite: { - name: 'Satellite', - url: 'https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}', - attribution: - 'Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community', - }, +/** + * Hoisted so every marker shares one icon instance: react-leaflet calls `setIcon` whenever the + * icon prop identity changes, which re-created all marker DOM on each render. + */ +const MARKER_ICON = L.divIcon({ + html: ` +
+
+
+ `, + className: 'custom-marker', + iconSize: [32, 32], + iconAnchor: [16, 32], + popupAnchor: [0, -32], +}); + +const SHAPE_STYLE = { + color: '#3b82f6', + weight: 2, + fillColor: '#3b82f6', + fillOpacity: 0.2, }; -const MapViewDynamic: React.FC = ({ json }) => { - const { t } = useTranslation(); - const [mounted, setMounted] = useState(false); - const [locations, setLocations] = useState([]); - const [isGeoJSON, setIsGeoJSON] = useState(false); - const [mapStyle, setMapStyle] = useState('dark'); +const shapePointToLayer = (_feature: unknown, latlng: L.LatLng) => + L.marker(latlng, { icon: MARKER_ICON }); + +/** Popup content is built with `textContent`, never HTML, so user JSON can never inject markup. */ +const bindShapePopup = (feature: any, layer: L.Layer): void => { + const container = document.createElement('div'); + container.className = 'text-sm'; + + const geometryType = toDisplayString(feature?.geometry?.type) || 'Shape'; + const title = document.createElement('div'); + title.className = 'font-bold mb-1'; + title.textContent = toDisplayString(feature?.properties?.name) || geometryType; + container.appendChild(title); + + const properties = feature?.properties; + if (properties && typeof properties === 'object') { + for (const [key, value] of Object.entries(properties).slice(0, 5)) { + const row = document.createElement('div'); + row.textContent = `${key.replace(/_/g, ' ')}: ${toDisplayString(value)}`; + container.appendChild(row); + } + } - useEffect(() => { - setMounted(true); - }, []); + const type = document.createElement('div'); + type.className = 'mt-1 text-xs opacity-70'; + type.textContent = geometryType; + container.appendChild(type); - useEffect(() => { - // Check if dark theme is active - const isDark = document.documentElement.classList.contains('dark'); - setMapStyle(isDark ? 'dark' : 'light'); - }, []); + layer.bindPopup(container); +}; + +const formatNumeric = (value: unknown): string => + typeof value === 'number' && Number.isFinite(value) + ? value.toLocaleString() + : toDisplayString(value); + +/** Well-known GeoJSON properties that get their own highlighted row in a marker popup. */ +const PROPERTY_ROWS: Array<{ + key: string; + label: string; + valueClass: string; + format?: (value: unknown) => string; +}> = [ + { key: 'country', label: 'Country', valueClass: 'text-emerald-600 dark:text-emerald-400' }, + { + key: 'population', + label: 'Population', + valueClass: 'text-blue-600 dark:text-blue-400', + format: formatNumeric, + }, + { key: 'state', label: 'State/Region', valueClass: 'text-purple-600 dark:text-purple-400' }, + { + key: 'gdp', + label: 'GDP', + valueClass: 'text-orange-600 dark:text-orange-400', + format: (value) => `$${toDisplayString(value)}B`, + }, + { key: 'timezone', label: 'Timezone', valueClass: 'text-indigo-600 dark:text-indigo-400' }, +]; + +/** + * `MapContainer`'s `center`/`zoom` are read once on mount in react-leaflet 5, so an in-place data + * change never recentred the map. This child re-fits from inside the map context instead; bounds + * arrive as four numbers so effect dependencies compare by value. + */ +const FitToData: React.FC<{ + minLat: number; + maxLat: number; + minLng: number; + maxLng: number; +}> = ({ minLat, maxLat, minLng, maxLng }) => { + const map = useMap(); useEffect(() => { - if (!json) return; - - // Check if it's GeoJSON format - if (json.type === 'FeatureCollection' && json.features) { - setIsGeoJSON(true); - const geoLocations: LocationData[] = []; - - json.features.forEach((feature: any, index: number) => { - if (feature.geometry && feature.geometry.coordinates) { - const coords = feature.geometry.coordinates; - if (feature.geometry.type === 'Point' && coords.length >= 2) { - geoLocations.push({ - lng: coords[0], - lat: coords[1], - data: feature, - path: `features[${index}]`, - name: feature.properties?.name || feature.properties?.title || `Feature ${index}`, - properties: feature.properties, - }); - } - } - }); - - setLocations(geoLocations); - } else if (json.type === 'Feature' && json.geometry) { - // Single GeoJSON feature - setIsGeoJSON(true); - if (json.geometry.type === 'Point' && json.geometry.coordinates) { - const coords = json.geometry.coordinates; - setLocations([ - { - lng: coords[0], - lat: coords[1], - data: json, - path: 'root', - name: json.properties?.name || 'Feature', - properties: json.properties, - }, - ]); + let frame = 0; + let attempts = 0; + + const applyView = () => { + // A container that has not been laid out yet reports a zero size, and fitBounds would + // silently commit a max-zoom view that nothing later corrects — so wait a few frames. + const size = map.getSize(); + if ((size.x === 0 || size.y === 0) && attempts < 10) { + attempts += 1; + frame = requestAnimationFrame(applyView); + return; } - } else { - // Regular JSON - search for location data - setIsGeoJSON(false); - const extractLocations = (obj: any, path: string = ''): LocationData[] => { - const locs: LocationData[] = []; - - if (obj && typeof obj === 'object') { - // Check for various location formats - if ( - (obj.lat && obj.lng) || - (obj.latitude && obj.longitude) || - (obj.lat && obj.lon) || - (obj.latitude && obj.lng) || - (obj.lat && obj.longitude) || - (obj.coordinates && Array.isArray(obj.coordinates) && obj.coordinates.length >= 2) || - (obj.location && typeof obj.location === 'object') - ) { - let lat = null; - let lng = null; - - if (obj.lat !== undefined && obj.lng !== undefined) { - lat = obj.lat; - lng = obj.lng; - } else if (obj.latitude !== undefined && obj.longitude !== undefined) { - lat = obj.latitude; - lng = obj.longitude; - } else if (obj.lat !== undefined && obj.lon !== undefined) { - lat = obj.lat; - lng = obj.lon; - } else if (obj.latitude !== undefined && obj.lng !== undefined) { - lat = obj.latitude; - lng = obj.lng; - } else if (obj.lat !== undefined && obj.longitude !== undefined) { - lat = obj.lat; - lng = obj.longitude; - } else if (obj.coordinates && Array.isArray(obj.coordinates)) { - lng = obj.coordinates[0]; - lat = obj.coordinates[1]; - } else if (obj.location) { - if (obj.location.lat !== undefined && obj.location.lng !== undefined) { - lat = obj.location.lat; - lng = obj.location.lng; - } else if ( - obj.location.latitude !== undefined && - obj.location.longitude !== undefined - ) { - lat = obj.location.latitude; - lng = obj.location.longitude; - } - } - - if ( - lat !== null && - lng !== null && - !isNaN(parseFloat(lat)) && - !isNaN(parseFloat(lng)) - ) { - locs.push({ - lat: parseFloat(lat), - lng: parseFloat(lng), - data: obj, - path, - name: obj.name || obj.city || obj.title || obj.label || obj.place || path, - }); - } - } - - // Recursive search - handle arrays properly - if (Array.isArray(obj)) { - obj.forEach((item, index) => { - if (typeof item === 'object' && item !== null) { - const newPath = path ? `${path}[${index}]` : `[${index}]`; - locs.push(...extractLocations(item, newPath)); - } - }); - } else { - Object.entries(obj).forEach(([key, value]) => { - if (typeof value === 'object' && value !== null) { - const newPath = path ? `${path}.${key}` : key; - locs.push(...extractLocations(value, newPath)); - } - }); - } - } - - return locs; - }; - - setLocations(extractLocations(json)); - } - }, [json]); - // Calculate center and bounds - const mapConfig = useMemo(() => { - if (locations.length === 0) { - return { center: [0, 0] as [number, number], zoom: 2 }; - } + if (minLat === maxLat && minLng === maxLng) { + // Zero-area bounds: fitBounds would slam to max zoom, so pick a sane single-pin view. + map.setView([minLat, minLng], 10); + return; + } - if (locations.length === 1) { - return { center: [locations[0].lat, locations[0].lng] as [number, number], zoom: 10 }; - } + map.fitBounds( + [ + [minLat, minLng], + [maxLat, maxLng], + ], + { padding: [40, 40] } + ); + }; + + applyView(); + return () => cancelAnimationFrame(frame); + }, [map, minLat, maxLat, minLng, maxLng]); + + return null; +}; - // Calculate bounds for multiple locations - const lats = locations.map((l) => l.lat); - const lngs = locations.map((l) => l.lng); - const minLat = Math.min(...lats); - const maxLat = Math.max(...lats); - const minLng = Math.min(...lngs); - const maxLng = Math.max(...lngs); - - const centerLat = (minLat + maxLat) / 2; - const centerLng = (minLng + maxLng) / 2; - - // Calculate appropriate zoom level based on bounds - const latDiff = maxLat - minLat; - const lngDiff = maxLng - minLng; - const maxDiff = Math.max(latDiff, lngDiff); - - let zoom = 2; - if (maxDiff < 0.01) zoom = 15; - else if (maxDiff < 0.1) zoom = 12; - else if (maxDiff < 1) zoom = 10; - else if (maxDiff < 10) zoom = 6; - else if (maxDiff < 50) zoom = 4; - else if (maxDiff < 100) zoom = 3; - - return { center: [centerLat, centerLng] as [number, number], zoom }; - }, [locations]); - - if (!json) { +const MarkerPopup: React.FC<{ location: GeoLocation; isGeoJSON: boolean }> = ({ + location, + isGeoJSON, +}) => { + const properties = + location.properties && typeof location.properties === 'object' ? location.properties : null; + + return ( +
+ {location.name && ( +
+ {location.name} +
+ )} + {properties && ( +
+ {PROPERTY_ROWS.map((row) => { + const raw = (properties as Record)[row.key]; + if (toDisplayString(raw) === '') return null; + return ( +
+ {row.label}: + + {row.format ? row.format(raw) : toDisplayString(raw)} + +
+ ); + })} +
+ )} + {location.data && !properties && ( +
+ {Object.entries(location.data) + .filter( + ([key]) => + ![ + 'lat', + 'lng', + 'lon', + 'long', + 'latitude', + 'longitude', + 'coordinates', + 'location', + ].includes(key.toLowerCase()) + ) + .slice(0, 5) + .map(([key, value]) => ( +
+ + {key.replace(/_/g, ' ')}: + + + {toDisplayString(value)} + +
+ ))} +
+ )} +
+
+ Coordinates +
+
+
+ Lat:{' '} + + {location.lat.toFixed(4)}° + +
+
+ Lng:{' '} + + {location.lng.toFixed(4)}° + +
+
+
+ {!isGeoJSON && ( +
+ Path: {location.path} +
+ )} +
+ ); +}; + +const MapViewDynamic: React.FC = ({ json, isValid }) => { + const { t } = useTranslation(); + const { theme } = useThemeContext(); + const [styleOverride, setStyleOverride] = useState(null); + // Read once during the first render: this component only ever runs in the browser + // (`MapView` loads it with `ssr: false`), so there is no server markup to mismatch. + const [tilesEnabled, setTilesEnabled] = useState(readTileConsent); + + // Follow the app theme until the user picks a style explicitly, so toggling the theme + // no longer leaves a dark basemap on a light page. + const mapStyle: MapStyleId = styleOverride ?? (theme === 'dark' ? 'dark' : 'light'); + const activeStyle = MAP_STYLES[mapStyle]; + + // The extractor walks the whole document looking for coordinates, so it is one + // of the recursions a deeply nested document kills. Guarded here rather than + // inside `extractGeoData` so the extractor stays a pure function. + const extraction = runDepthGuarded(() => extractGeoData(json)); + const geo = extraction.ok ? extraction.value : EMPTY_EXTRACTION; + const { locations, totalLocations, shapes, shapeTypes, isGeoJSON, bounds } = geo; + const initialView = viewFromBounds(bounds); + + const enableTiles = () => { + storeTileConsent(); + setTilesEnabled(true); + }; + + // `!json` would call `0`, `false`, `""` and `null` "no data"; they are all valid documents. + const hasDocument = isValid ?? (json !== null && json !== undefined); + + if (!hasDocument) { return (

- {mounted ? t('map.noData') || 'No JSON data available' : 'No JSON data available'} + {t('map.noData', { defaultValue: 'No JSON data available' })}

); } - if (locations.length === 0) { + if (!extraction.ok) { + return ( +
+ +
+ ); + } + + if (locations.length === 0 && shapes.length === 0) { return (

- {mounted ? t('map.noGeoData') : 'No geographic data found'} + {t('map.noGeoData', { defaultValue: 'No geographic data found' })}

- {mounted - ? t('map.supportedFormats') || 'Map view supports the following formats:' - : 'Map view supports the following formats:'} + {t('map.supportedFormats', { + defaultValue: 'Map view supports the following formats:', + })}

  • • GeoJSON FeatureCollection or Feature
    • @@ -282,10 +364,10 @@ const MapViewDynamic: React.FC = ({ json }) => {

- {mounted - ? t('map.tip') || - 'Tip: For GeoJSON, ensure geometry.type is "Point" and coordinates are [longitude, latitude]' - : 'Tip: For GeoJSON, ensure geometry.type is "Point" and coordinates are [longitude, latitude]'} + {t('map.tip', { + defaultValue: + 'Tip: For GeoJSON, ensure geometry.type is "Point" and coordinates are [longitude, latitude]', + })}

@@ -296,204 +378,133 @@ const MapViewDynamic: React.FC = ({ json }) => { return (
{/* Location counter */} -
-

- {locations.length}{' '} - {mounted - ? locations.length === 1 - ? t('map.locationSingular') || 'location found' - : t('map.locationPlural') || 'locations found' - : `location${locations.length !== 1 ? 's' : ''} found`} -

+
+ {locations.length > 0 && ( +

+ {locations.length}{' '} + {locations.length === 1 + ? t('map.locationSingular', { defaultValue: 'location found' }) + : t('map.locationPlural', { defaultValue: 'locations found' })} +

+ )} + {totalLocations > locations.length && ( +

+ {t('map.markerCap', { + defaultValue: 'Showing the first {{shown}} of {{total}} — the rest are not plotted.', + shown: locations.length, + total: totalLocations, + })} +

+ )} + {shapes.length > 0 && ( +

+ {t('map.shapesFound', { + defaultValue: '{{total}} shape(s) drawn: {{types}}', + total: shapes.length, + types: shapeTypes.join(', '), + })} +

+ )} {isGeoJSON && (

- {mounted - ? t('map.geoJSONDetected') || 'GeoJSON format detected' - : 'GeoJSON format detected'} + {t('map.geoJSONDetected', { defaultValue: 'GeoJSON format detected' })}

)}
- {/* Map style switcher - positioned below zoom controls */} -
- - + {/* Tile opt-in / map style switcher - positioned below zoom controls */} +
+ {tilesEnabled ? ( + <> + + + + ) : ( +
+

+ {t('map.tilesOffTitle', { defaultValue: 'Background map is off' })} +

+

+ {t('map.tilesOffBody', { + defaultValue: + 'Your JSON stays in the browser. Loading tiles sends map image requests to the third parties listed here, for any map style you pick, and reveals the area you are viewing and your IP address to them.', + })} +

+ {/* Every provider any style can reach, not just the selected one: one grant covers + them all, so the panel has to name them all. */} +
    + {TILE_PROVIDERS.map(({ provider, host }) => ( +
  • + {provider}{' '} + {host} +
    • + ))} +
+ +
+ )}
- - - {locations.map((location, index) => { - // Create custom icon with better colors - const customIcon = L.divIcon({ - html: ` -
-
-
- `, - className: 'custom-marker', - iconSize: [32, 32], - iconAnchor: [16, 32], - popupAnchor: [0, -32], - }); - - return ( - - -
- {location.name && ( -
- {location.name} -
- )} - {location.properties && ( -
- {location.properties.country && ( -
- - Country: - - - {location.properties.country} - -
- )} - {location.properties.population && ( -
- - Population: - - - {location.properties.population.toLocaleString()} - -
- )} - {location.properties.state && ( -
- - State/Region: - - - {location.properties.state} - -
- )} - {location.properties.gdp && ( -
- - GDP: - - - ${location.properties.gdp}B - -
- )} - {location.properties.timezone && ( -
- - Timezone: - - - {location.properties.timezone} - -
- )} -
- )} - {/* Show other relevant data fields */} - {location.data && !location.properties && ( -
- {Object.entries(location.data) - .filter( - ([key]) => - ![ - 'lat', - 'lng', - 'latitude', - 'longitude', - 'coordinates', - 'location', - ].includes(key.toLowerCase()) - ) - .slice(0, 5) - .map(([key, value]) => ( -
- - {key.replace(/_/g, ' ')}: - - - {typeof value === 'object' ? JSON.stringify(value) : String(value)} - -
- ))} -
- )} -
-
- Coordinates -
-
-
- Lat:{' '} - - {location.lat.toFixed(4)}° - -
-
- Lng:{' '} - - {location.lng.toFixed(4)}° - -
-
-
- {!isGeoJSON && ( -
- Path:{' '} - - {location.path} - -
- )} -
- - - ); - })} + {tilesEnabled && } + + {bounds && ( + + )} + + {shapes.length > 0 && ( + + )} + + {locations.map((location, index) => ( + + + + + + ))}
); diff --git a/src/components/MonacoEditorWrapper.tsx b/src/components/MonacoEditorWrapper.tsx deleted file mode 100644 index 6158efa..0000000 --- a/src/components/MonacoEditorWrapper.tsx +++ /dev/null @@ -1,75 +0,0 @@ -'use client'; - -import Editor, { loader } from '@monaco-editor/react'; -import type React from 'react'; -import { useEffect } from 'react'; - -// Load Monaco from the same origin (copied into /public/monaco at build time) -// instead of a third-party CDN. This keeps user data private (no request leaves -// the browser to jsdelivr/unpkg), works offline, and removes a supply-chain risk. -loader.config({ - paths: { - vs: '/monaco/vs', - }, - 'vs/nls': { - availableLanguages: {}, - }, -} as any); - -interface MonacoEditorWrapperProps { - height: string; - defaultLanguage: string; - theme: string; - value: string; - onChange: (value: string | undefined) => void; - onMount: (editor: any, monaco: any) => void; - options: any; - loading?: React.ReactNode; -} - -const MonacoEditorWrapper: React.FC = (props) => { - useEffect(() => { - // Disable workers globally when component mounts - this runs in synchronous mode - if (typeof window !== 'undefined' && !(window as any).MonacoEnvironment) { - // Only set if not already set by suppress-monaco-warnings.js - (window as any).MonacoEnvironment = { - // Don't provide getWorker at all - this forces synchronous mode - getWorkerUrl: (_: string, label: string) => { - // Return data URL to prevent network requests - return 'data:text/javascript;charset=utf-8,'; - }, - }; - } - }, []); - - return ( - { - // Disable JSON validation to avoid worker requirements - try { - monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ - validate: false, - comments: 'ignore', - trailingCommas: 'ignore', - schemas: [], - }); - } catch (e) { - console.info('JSON defaults configuration skipped'); - } - - // Disable TypeScript/JavaScript workers if they exist - try { - if (monaco.languages.typescript) { - monaco.languages.typescript.typescriptDefaults.setEagerModelSync(true); - monaco.languages.typescript.javascriptDefaults.setEagerModelSync(true); - } - } catch (e) { - console.info('TypeScript defaults configuration skipped'); - } - }} - /> - ); -}; - -export default MonacoEditorWrapper; diff --git a/src/components/SearchView.tsx b/src/components/SearchView.tsx index 9dc9764..1292cea 100644 --- a/src/components/SearchView.tsx +++ b/src/components/SearchView.tsx @@ -1,31 +1,37 @@ 'use client'; import type React from 'react'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import { + analyzeJsonStructure, + buildSuggestions, + DEFAULT_RESULT_LIMIT, + highlightSegments, + type RegexIssue, + type SearchMode, + type SearchOutcome, + type SearchResult, + type SearchTarget, + type StructureAnalysis, + searchJson, +} from '@/utils/jsonSearch'; +import { MAX_JSON_DEPTH } from '@/utils/jsonWalk'; interface SearchViewProps { json: any; + /** + * Whether `json` holds a parsed document at all. + * + * Carried explicitly rather than inferred from the value, because `0`, + * `false`, `""` and `null` are all perfectly valid JSON documents that a + * truthiness test reports as "nothing loaded". Optional while the workbench + * is being threaded: with the prop absent this falls back to the old + * inference, so behaviour is unchanged until it arrives. + */ + isValid?: boolean; } -interface SearchResult { - type: 'key' | 'value' | 'both'; - path: string; - key: string; - value: any; - depth: number; - dataType: string; -} - -interface WordFrequency { - word: string; - count: number; - percentage: number; -} - -type SearchMode = 'contains' | 'exact' | 'starts' | 'ends' | 'regex'; -type SearchTarget = 'both' | 'keys' | 'values'; - interface TreeNode { path: string; results: SearchResult[]; @@ -33,6 +39,80 @@ interface TreeNode { expanded?: boolean; } +const TYPE_META: Record = { + string: { icon: '"', text: 'text-green-500', badge: 'text-green-500 bg-green-500/10' }, + number: { icon: '#', text: 'text-blue-500', badge: 'text-blue-500 bg-blue-500/10' }, + boolean: { icon: '⊤', text: 'text-purple-500', badge: 'text-purple-500 bg-purple-500/10' }, + object: { icon: '{}', text: 'text-orange-500', badge: 'text-orange-500 bg-orange-500/10' }, + array: { icon: '[]', text: 'text-yellow-500', badge: 'text-yellow-500 bg-yellow-500/10' }, + null: { icon: '∅', text: 'text-gray-500', badge: 'text-gray-500 bg-gray-500/10' }, +}; + +const UNKNOWN_TYPE_META = { + icon: '?', + text: 'text-gray-400', + badge: 'text-gray-400 bg-gray-400/10', +}; + +const typeMeta = (type: string) => TYPE_META[type] ?? UNKNOWN_TYPE_META; +const getTypeIcon = (type: string) => typeMeta(type).icon; +const getTypeTextColor = (type: string) => typeMeta(type).text; +const getTypeBadgeColor = (type: string) => typeMeta(type).badge; + +const EMPTY_RESULTS: SearchResult[] = []; + +const EMPTY_ANALYSIS: StructureAnalysis = { + paths: [], + pathsTruncated: false, + totalKeys: 0, + totalValues: 0, + maxDepth: 0, + depthExceeded: false, + dataTypes: {}, + wordFrequencies: [], +}; + +// The search engine bounds its own recursion rather than throwing, so this view +// keeps working on a deeply nested document and only has to say what it skipped. +const DEPTH_NOTICE_FALLBACK = + 'Branches nested more than {{limit}} levels deep were skipped, so some results may be missing.'; + +/** + * Fills `{{name}}` placeholders in an English fallback, for the first render — + * before i18next has resolved the browser language and can do it itself. + */ +const interpolate = (text: string, vars?: Record) => + vars ? text.replace(/{{(\w+)}}/g, (whole, name) => String(vars[name] ?? whole)) : text; + +/** + * English copy for a pattern the search engine refused. + * + * The engine returns a code rather than a sentence: it has no locale, and these + * twelve translations belong with the component that renders them. Every one of + * them says the pattern was *not run* — none of them claims the patterns that + * do run have been checked and found safe, because they have not been. + */ +const REGEX_ISSUE_COPY: Record = { + 'too-long': { + key: 'search.regexTooLong', + fallback: 'This pattern is {{length}} characters long; the limit is {{limit}}. It was not run.', + }, + 'nested-quantifier': { + key: 'search.regexNestedQuantifier', + fallback: + 'This pattern nests one quantifier inside another, like (a+)+, a shape that can take effectively forever to match. It was not run.', + }, + 'too-slow': { + key: 'search.regexTooSlow', + fallback: + 'This pattern was too slow on a short test string, so it was not run. Try a simpler one.', + }, + invalid: { + key: 'search.regexInvalid', + fallback: 'This is not a valid regular expression.', + }, +}; + // Tree Result View Component const TreeResultView: React.FC<{ results: SearchResult[]; @@ -89,44 +169,6 @@ const TreeResultView: React.FC<{ setExpandedNodes(newExpanded); }; - const getTypeIcon = (type: string) => { - switch (type) { - case 'string': - return '"'; - case 'number': - return '#'; - case 'boolean': - return '⊤'; - case 'object': - return '{}'; - case 'array': - return '[]'; - case 'null': - return '∅'; - default: - return '?'; - } - }; - - const getTypeColor = (type: string) => { - switch (type) { - case 'string': - return 'text-green-500'; - case 'number': - return 'text-blue-500'; - case 'boolean': - return 'text-purple-500'; - case 'object': - return 'text-orange-500'; - case 'array': - return 'text-yellow-500'; - case 'null': - return 'text-gray-500'; - default: - return 'text-gray-400'; - } - }; - const renderTreeNode = (node: TreeNode, key: string, level: number = 0): React.ReactElement => { const hasChildren = node.children.size > 0; const isExpanded = expandedNodes.has(node.path); @@ -170,7 +212,7 @@ const TreeResultView: React.FC<{ style={{ paddingLeft: `${indent + 24}px` }} > {getTypeIcon(result.dataType)} @@ -247,7 +289,7 @@ const TreeResultView: React.FC<{ }`} > {getTypeIcon(result.dataType)} @@ -270,7 +312,7 @@ const TreeResultView: React.FC<{ ); }; -const SearchView: React.FC = ({ json }) => { +const SearchView: React.FC = ({ json, isValid }) => { const { t } = useTranslation(); const [mounted, setMounted] = useState(false); @@ -278,9 +320,8 @@ const SearchView: React.FC = ({ json }) => { setMounted(true); }, []); const [searchQuery, setSearchQuery] = useState(''); - const [searchResults, setSearchResults] = useState([]); - const [allPaths, setAllPaths] = useState([]); - const [wordFrequencies, setWordFrequencies] = useState([]); + const [searchOutcome, setSearchOutcome] = useState(null); + const [analysis, setAnalysis] = useState(EMPTY_ANALYSIS); const [selectedResult, setSelectedResult] = useState(null); const [viewMode, setViewMode] = useState<'list' | 'tree' | 'visual'>('list'); const [showAdvancedFilters, setShowAdvancedFilters] = useState(false); @@ -305,40 +346,89 @@ const SearchView: React.FC = ({ json }) => { const [pathPattern, setPathPattern] = useState(''); const [excludePattern, setExcludePattern] = useState(''); - const [isAnalyzing, setIsAnalyzing] = useState(false); const [searchHistory, setSearchHistory] = useState([]); const [showSuggestions, setShowSuggestions] = useState(false); - const [regexError, setRegexError] = useState(''); - const [dataInsights, setDataInsights] = useState({ - totalKeys: 0, - totalValues: 0, - maxDepth: 0, - dataTypes: {} as Record, + + // `0`, `false`, `""` and `null` are documents; only the absence of one is not. + const hasDocument = isValid ?? (json !== null && json !== undefined); + + const searchResults = searchOutcome?.results ?? EMPTY_RESULTS; + const totalMatches = searchOutcome?.totalMatches ?? 0; + const resultsTruncated = searchOutcome?.truncated ?? false; + const budgetExhausted = searchOutcome?.budgetExhausted ?? false; + // Either walk can hit the ceiling: the search itself, or the structure pass + // behind the insight panel. + const depthExceeded = (searchOutcome?.depthExceeded ?? false) || analysis.depthExceeded; + + // Translated copy with an English fallback, so the first render matches what + // the server produced before i18next has resolved the browser language. + const label = (key: string, fallback: string, vars?: Record) => + mounted ? t(key, { defaultValue: fallback, ...vars }) : interpolate(fallback, vars); + + const depthNotice = label('depth.searchNotice', DEPTH_NOTICE_FALLBACK, { + limit: MAX_JSON_DEPTH, }); - // Analyze JSON structure on mount + // The trailing '+' says the walk gave up before the end, so the number is a + // floor. Built here rather than passed as its own placeholder, which would + // leave every translator a bare '+' to find a home for. `count` is avoided as + // a placeholder name throughout: i18next reserves it for pluralisation. + const matchCountLabel = budgetExhausted ? `${totalMatches}+` : String(totalMatches); + + const regexIssueText = (issue: RegexIssue | null | undefined): string => { + if (!issue) return ''; + const { key, fallback } = REGEX_ISSUE_COPY[issue.code]; + const message = label(key, fallback, { length: issue.length ?? 0, limit: issue.limit ?? 0 }); + // The engine's own wording for an unparseable pattern. It arrives in + // English from the JS runtime and there is nothing to translate it against, + // so it is appended rather than folded into the sentence. + return issue.code === 'invalid' && issue.detail ? `${message} (${issue.detail})` : message; + }; + + const regexError = regexIssueText(searchOutcome?.queryError); + const pathPatternError = regexIssueText(searchOutcome?.pathPatternError); + const excludePatternError = regexIssueText(searchOutcome?.excludePatternError); + const { paths: allPaths, wordFrequencies, dataTypes } = analysis; + + // Structure analysis is a whole-document walk, so it is kept in state and + // recomputed only when the document itself changes - never per keystroke. useEffect(() => { - if (json) { - analyzeJsonStructure(); - } - }, [json]); + setAnalysis(hasDocument ? analyzeJsonStructure(json) : EMPTY_ANALYSIS); + }, [json, hasDocument]); // Real-time search with debounce useEffect(() => { const timer = setTimeout(() => { - if (searchQuery) { - performSearch(); - setSearchHistory((prev) => - prev.includes(searchQuery) ? prev : [searchQuery, ...prev].slice(0, 10) - ); - } else { - setSearchResults([]); + if (!searchQuery.trim() || !hasDocument) { + setSearchOutcome(null); + setSelectedResult(null); + return; } + + setSearchOutcome( + searchJson(json, { + query: searchQuery, + mode: searchMode, + target: searchTarget, + caseSensitive, + maxDepth, + minLength, + maxLength, + pathPattern, + excludePattern, + filters: activeFilters, + resultLimit: DEFAULT_RESULT_LIMIT, + }) + ); + // Selection is an index into the previous result set, so a new search + // must drop it rather than leave a stale card highlighted. + setSelectedResult(null); }, 200); return () => clearTimeout(timer); }, [ json, + hasDocument, searchQuery, activeFilters, searchMode, @@ -351,256 +441,26 @@ const SearchView: React.FC = ({ json }) => { excludePattern, ]); - const analyzeJsonStructure = () => { - setIsAnalyzing(true); - const paths: string[] = []; - const words: Record = {}; - const typeCount: Record = {}; - let totalKeys = 0; - let totalValues = 0; - let maxDepthFound = 0; - - const analyze = (obj: any, path: string = '', depth: number = 0) => { - maxDepthFound = Math.max(maxDepthFound, depth); - - if (Array.isArray(obj)) { - typeCount.array = (typeCount.array || 0) + 1; - obj.forEach((item, index) => { - analyze(item, `${path}[${index}]`, depth + 1); - }); - } else if (obj && typeof obj === 'object') { - typeCount.object = (typeCount.object || 0) + 1; - Object.entries(obj).forEach(([key, value]) => { - totalKeys++; - paths.push(`${path}${path ? '.' : ''}${key}`); - - // Extract words for frequency analysis - key.split(/[^a-zA-Z0-9]+/).forEach((word) => { - if (word.length > 2) { - words[word.toLowerCase()] = (words[word.toLowerCase()] || 0) + 1; - } - }); - - if (typeof value === 'string') { - totalValues++; - typeCount.string = (typeCount.string || 0) + 1; - value.split(/[^a-zA-Z0-9]+/).forEach((word) => { - if (word.length > 2) { - words[word.toLowerCase()] = (words[word.toLowerCase()] || 0) + 1; - } - }); - } else if (typeof value === 'number') { - totalValues++; - typeCount.number = (typeCount.number || 0) + 1; - } else if (typeof value === 'boolean') { - totalValues++; - typeCount.boolean = (typeCount.boolean || 0) + 1; - } else if (value === null) { - totalValues++; - typeCount.null = (typeCount.null || 0) + 1; - } - - analyze(value, `${path}${path ? '.' : ''}${key}`, depth + 1); - }); - } - }; - - analyze(json); - - // Calculate word frequencies - const totalWords = Object.values(words).reduce((a, b) => a + b, 0); - const frequencies = Object.entries(words) - .map(([word, count]) => ({ - word, - count, - percentage: (count / totalWords) * 100, - })) - .sort((a, b) => b.count - a.count) - .slice(0, 30); - - setAllPaths(paths); - setWordFrequencies(frequencies); - setDataInsights({ - totalKeys, - totalValues, - maxDepth: maxDepthFound, - dataTypes: typeCount, - }); - setIsAnalyzing(false); + // Only committed searches (Enter, a suggestion, an insight chip) are + // remembered - not every debounced keystroke on the way there. + const rememberSearch = (term: string) => { + const trimmed = term.trim(); + if (!trimmed) return; + setSearchHistory((prev) => (prev.includes(trimmed) ? prev : [trimmed, ...prev].slice(0, 10))); }; - const testRegex = (pattern: string): RegExp | null => { - try { - return new RegExp(pattern, caseSensitive ? 'g' : 'gi'); - } catch (e) { - setRegexError(e instanceof Error ? e.message : 'Invalid regex'); - return null; - } + const commitSearch = (term: string) => { + setSearchQuery(term); + setShowSuggestions(false); + rememberSearch(term); }; - const matchesSearch = (text: string, query: string): boolean => { - const targetText = caseSensitive ? text : text.toLowerCase(); - const searchText = caseSensitive ? query : query.toLowerCase(); - - switch (searchMode) { - case 'exact': - return targetText === searchText; - case 'starts': - return targetText.startsWith(searchText); - case 'ends': - return targetText.endsWith(searchText); - case 'regex': { - const regex = testRegex(query); - return regex ? regex.test(text) : false; - } - case 'contains': - default: - return targetText.includes(searchText); - } - }; - - const performSearch = () => { - if (!json || !searchQuery.trim()) { - setSearchResults([]); - setRegexError(''); - return; - } - - // Clear regex error if not in regex mode - if (searchMode !== 'regex') { - setRegexError(''); - } - - const results: SearchResult[] = []; - const query = searchQuery; - - // Test regex validity if in regex mode - if (searchMode === 'regex') { - const regex = testRegex(query); - if (!regex) return; - } - - // Compile path pattern regex if provided - let pathRegex: RegExp | null = null; - if (pathPattern) { - try { - pathRegex = new RegExp(pathPattern, 'i'); - } catch { - // Invalid path pattern, skip - } - } - - // Compile exclude pattern regex if provided - let excludeRegex: RegExp | null = null; - if (excludePattern) { - try { - excludeRegex = new RegExp(excludePattern, 'i'); - } catch { - // Invalid exclude pattern, skip - } - } - - const search = (obj: any, path: string = '', depth: number = 0) => { - // Check max depth - if (maxDepth !== -1 && depth > maxDepth) return; - - // Check path pattern - if (pathRegex && !pathRegex.test(path)) return; - - // Check exclude pattern - if (excludeRegex && excludeRegex.test(path)) return; - - if (Array.isArray(obj)) { - if (!activeFilters.arrays) return; - obj.forEach((item, index) => { - search(item, `${path}[${index}]`, depth + 1); - }); - } else if (obj && typeof obj === 'object') { - if (!activeFilters.objects) return; - Object.entries(obj).forEach(([key, value]) => { - const currentPath = `${path}${path ? '.' : ''}${key}`; - let keyMatch = false; - let valueMatch = false; - let dataType = 'unknown'; - - // Check if we should search in keys - if (searchTarget === 'both' || searchTarget === 'keys') { - keyMatch = matchesSearch(key, query); - } - - // Check if we should search in values - if (searchTarget === 'both' || searchTarget === 'values') { - if (typeof value === 'string') { - if (!activeFilters.strings) return; - - // Check length filters - if (minLength > 0 && value.length < minLength) return; - if (maxLength !== -1 && value.length > maxLength) return; - - valueMatch = matchesSearch(value, query); - dataType = 'string'; - } else if (typeof value === 'number') { - if (!activeFilters.numbers) return; - valueMatch = matchesSearch(String(value), query); - dataType = 'number'; - } else if (typeof value === 'boolean') { - if (!activeFilters.booleans) return; - valueMatch = matchesSearch(String(value), query); - dataType = 'boolean'; - } else if (value === null) { - if (!activeFilters.nulls) return; - valueMatch = matchesSearch('null', query); - dataType = 'null'; - } else if (Array.isArray(value)) { - dataType = 'array'; - } else if (typeof value === 'object') { - dataType = 'object'; - } - } - - if (keyMatch || valueMatch) { - results.push({ - type: keyMatch && valueMatch ? 'both' : keyMatch ? 'key' : 'value', - path: currentPath, - key, - value, - depth, - dataType, - }); - } - - if (typeof value === 'object' && value !== null) { - search(value, currentPath, depth + 1); - } - }); - } - }; - - search(json); - setSearchResults(results); - }; - - const getSuggestions = useMemo(() => { - if (!searchQuery) return []; - - const query = searchQuery.toLowerCase(); - const pathSuggestions = allPaths - .filter((path) => path.toLowerCase().includes(query)) - .slice(0, 5); - - const wordSuggestions = wordFrequencies - .filter((w) => w.word.startsWith(query)) - .slice(0, 5) - .map((w) => w.word); - - return Array.from(new Set([...pathSuggestions, ...wordSuggestions])).slice(0, 8); - }, [searchQuery, allPaths, wordFrequencies]); + const suggestions = buildSuggestions(searchQuery, allPaths, wordFrequencies); const clearSearch = () => { setSearchQuery(''); - setSearchResults([]); + setSearchOutcome(null); setSelectedResult(null); - setRegexError(''); }; const resetFilters = () => { @@ -629,62 +489,20 @@ const SearchView: React.FC = ({ json }) => { })); }; - const getTypeIcon = (type: string) => { - switch (type) { - case 'string': - return '"'; - case 'number': - return '#'; - case 'boolean': - return '⊤'; - case 'object': - return '{}'; - case 'array': - return '[]'; - case 'null': - return '∅'; - default: - return '?'; - } - }; - - const getTypeColor = (type: string) => { - switch (type) { - case 'string': - return 'text-green-500 bg-green-500/10'; - case 'number': - return 'text-blue-500 bg-blue-500/10'; - case 'boolean': - return 'text-purple-500 bg-purple-500/10'; - case 'object': - return 'text-orange-500 bg-orange-500/10'; - case 'array': - return 'text-yellow-500 bg-yellow-500/10'; - case 'null': - return 'text-gray-500 bg-gray-500/10'; - default: - return 'text-gray-400 bg-gray-400/10'; - } - }; - const highlightMatch = (text: string) => { - if (!searchQuery || searchMode === 'regex') return text; - - const parts = text.split( - new RegExp(`(${searchQuery.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi') - ); + const segments = highlightSegments(text, searchQuery, searchMode, caseSensitive); return ( <> - {parts.map((part, i) => - part.toLowerCase() === searchQuery.toLowerCase() ? ( + {segments.map((segment, i) => + segment.match ? ( - {part} + {segment.text} ) : ( - {part} + {segment.text} ) )} @@ -692,7 +510,10 @@ const SearchView: React.FC = ({ json }) => { }; const copyToClipboard = (text: string) => { - navigator.clipboard.writeText(text); + // Copying is a convenience: the clipboard can be unavailable (insecure + // context, denied permission), so handle the rejection instead of leaving + // an unhandled promise behind. + navigator.clipboard?.writeText(text).catch(() => undefined); }; const exportResults = () => { @@ -710,7 +531,12 @@ const SearchView: React.FC = ({ json }) => { dataTypes: activeFilters, }, timestamp: new Date().toISOString(), + // resultsCount is what the export contains; totalMatches is what the + // document holds, which is larger when the cap kicked in. resultsCount: searchResults.length, + totalMatches, + truncated: resultsTruncated, + searchStoppedEarly: budgetExhausted, results: searchResults.map((r) => ({ path: r.path, key: r.key, @@ -726,7 +552,11 @@ const SearchView: React.FC = ({ json }) => { const a = document.createElement('a'); a.href = url; a.download = `search-results-${Date.now()}.json`; + // The anchor has to be in the document for the click to start a download + // in every browser, and it must outlive the click before the URL is freed. + document.body.appendChild(a); a.click(); + a.remove(); URL.revokeObjectURL(url); }; @@ -740,9 +570,9 @@ const SearchView: React.FC = ({ json }) => { {mounted ? t('search.smartSearch') : 'Smart Search & Discovery'}

- {dataInsights.totalKeys} {mounted ? t('search.keys') : 'keys'} •{' '} - {dataInsights.totalValues} {mounted ? t('search.values') : 'values'} •{' '} - {mounted ? t('search.depth') : 'Depth'}: {dataInsights.maxDepth} + {analysis.totalKeys} {mounted ? t('search.keys') : 'keys'} • {analysis.totalValues}{' '} + {mounted ? t('search.values') : 'values'} • {mounted ? t('search.depth') : 'Depth'}:{' '} + {analysis.maxDepth}

@@ -782,6 +612,9 @@ const SearchView: React.FC = ({ json }) => { setSearchQuery(e.target.value); setShowSuggestions(true); }} + onKeyDown={(e) => { + if (e.key === 'Enter') commitSearch(searchQuery); + }} onFocus={() => setShowSuggestions(true)} onBlur={() => setTimeout(() => setShowSuggestions(false), 200)} placeholder={mounted ? t('search.placeholder') : 'Search keys or values...'} @@ -817,23 +650,36 @@ const SearchView: React.FC = ({ json }) => { )}
- {/* Regex Error */} - {regexError && searchMode === 'regex' && ( -
Regex Error: {regexError}
+ {/* Pattern errors */} + {(regexError || pathPatternError || excludePatternError) && ( +
+ {regexError && ( +
+ {label('search.queryPatternLabel', 'Search pattern')}: {regexError} +
+ )} + {pathPatternError && ( +
+ {label('search.pathPatternLabel', 'Path pattern')}: {pathPatternError} +
+ )} + {excludePatternError && ( +
+ {label('search.excludePatternLabel', 'Exclude pattern')}: {excludePatternError} +
+ )} +
)} {/* Search Suggestions */} - {showSuggestions && getSuggestions.length > 0 && ( + {showSuggestions && suggestions.length > 0 && (

Suggestions

- {getSuggestions.map((suggestion, i) => ( + {suggestions.map((suggestion, i) => (
-
- {Object.entries(json).map(([key, value]) => renderNode(key, value, `root.${key}`))} +
+ {entriesOf(json).map(([segment, value]) => renderNode([segment], value, 0))}
); diff --git a/src/components/seo/JsonLd.tsx b/src/components/seo/JsonLd.tsx new file mode 100644 index 0000000..05aaea5 --- /dev/null +++ b/src/components/seo/JsonLd.tsx @@ -0,0 +1,25 @@ +/** + * Renders a structured-data block as `` ends the block and + * everything after it is parsed as markup. Escaping `<` as `<` is valid + * inside a JSON string, decodes back to the same character, and makes that + * impossible. + * + * Every schema in this app is currently built from static literals, so there is + * no live injection path. This exists so that stays true by construction rather + * than by remembering — the moment a page title or a route param reaches a + * schema, twelve hand-written copies of the raw pattern would each have been a + * hole. + */ +export default function JsonLd({ data }: { data: unknown }) { + return ( + '; + const leaf = childAt(buildTree({ [key]: value }, '{ }', '$'), 0); + const model = buildTooltipModel(leaf) as Extract; + + expect(model.path).toBe(`$.${key}`); + // Short enough to survive whole: the tags come back exactly as pasted. + expect(value.length).toBeLessThanOrEqual(PREVIEW_MAX_LENGTH); + expect(model.value).toBe(`"${value}"`); + + for (const field of Object.values(model)) { + // No wrapper markup: the old implementation emitted `
` + // around every one of these fields. + expect(field).not.toMatch(/ { + const tree = buildTree({ 'k': [1] }, '{ }', '$'); + const model = buildTooltipModel(childAt(tree, 0)); + expect(model.path).toBe('$.k'); + expect(model).not.toHaveProperty('html'); + expect(JSON.stringify(model)).not.toContain(' { + it('reports a 100,000-level document instead of throwing RangeError', () => { + const deep = nestDeeply(100_000, 'array'); + expect(runDepthGuarded(() => buildTree(deep, 'root', '$'))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + }); + + it('builds a document that sits exactly on the limit', () => { + const atLimit = nestDeeply(MAX_JSON_DEPTH, 'object'); + const built = runDepthGuarded(() => buildTree(atLimit, 'root', '$')); + expect(built.ok).toBe(true); + }); + + it('leaves wide-but-shallow documents alone', () => { + const wide = Array.from({ length: 2_000 }, (_, i) => i); + const tree = buildTree(wide, 'root', '$'); + expect(tree.children).toHaveLength(2_000); + expect(childAt(tree, 1_999).path).toBe('$[1999]'); + }); +}); diff --git a/src/utils/graphData.ts b/src/utils/graphData.ts new file mode 100644 index 0000000..255da83 --- /dev/null +++ b/src/utils/graphData.ts @@ -0,0 +1,106 @@ +/** + * Pure data helpers behind the D3 graph view. + * + * These live outside the component on purpose: everything here is fed straight + * from user-supplied JSON (arbitrary keys, arbitrary string values), so it is + * the part that most needs unit tests. Nothing here produces markup — the + * tooltip builder returns a plain data model that the component renders with + * DOM nodes and `.text()`, never with `.html()`. + */ + +import { assertJsonDepth } from './jsonWalk'; + +export type JsonNodeType = 'object' | 'array' | 'value'; + +export interface JsonNode { + name: string; + type: JsonNodeType; + path: string; + value?: unknown; + valueType?: string; + children?: JsonNode[]; +} + +/** Strings longer than this are truncated in node labels and tooltips. */ +export const PREVIEW_MAX_LENGTH = 28; + +/** + * Render a leaf value as display text. Strings keep their quotes and are + * truncated with an ellipsis; everything else is stringified as-is. + * + * The result is plain text, never escaped and never markup — callers must + * insert it via `textContent` / d3 `.text()`. + */ +export function previewValue(value: unknown): string { + if (value === null) return 'null'; + if (typeof value === 'string') { + return value.length > PREVIEW_MAX_LENGTH + ? `"${value.slice(0, PREVIEW_MAX_LENGTH)}…"` + : `"${value}"`; + } + return String(value); +} + +/** + * Turn parsed JSON into the hierarchy d3 lays out, tagging every node with the + * JSONPath-ish path shown in the tooltip (`$.users[0].name`). + * + * Throws `JsonDepthLimitError` past `MAX_JSON_DEPTH`; GraphView runs it inside + * `runDepthGuarded` and shows the "nested too deeply" panel instead. Capping + * here is also what keeps d3 safe: `d3.hierarchy` and the tree layout only ever + * see the shape this function produced. + */ +export function buildTree(data: unknown, name: string, path: string, depth = 0): JsonNode { + assertJsonDepth(depth); + if (Array.isArray(data)) { + return { + name, + type: 'array', + path, + children: data.map((v, i) => buildTree(v, String(i), `${path}[${i}]`, depth + 1)), + }; + } + if (data !== null && typeof data === 'object') { + return { + name, + type: 'object', + path, + children: Object.entries(data).map(([k, v]) => buildTree(v, k, `${path}.${k}`, depth + 1)), + }; + } + return { + name, + type: 'value', + path, + value: data, + valueType: data === null ? 'null' : typeof data, + }; +} + +/** + * The text the hover tooltip shows, as data rather than a string of HTML. + * + * `path` and `value` are raw user input and are returned verbatim: escaping + * them here would be the wrong fix (it would corrupt legitimate values like + * `"a < b"`). Safety comes from the render side using `.text()`. + */ +export type GraphTooltipModel = + | { path: string; kind: 'value'; value: string; valueType: string } + | { path: string; kind: 'container'; summary: string }; + +export function buildTooltipModel(node: JsonNode): GraphTooltipModel { + if (node.type === 'value') { + return { + path: node.path, + kind: 'value', + value: previewValue(node.value), + valueType: node.valueType ?? '', + }; + } + const count = node.children ? node.children.length : 0; + return { + path: node.path, + kind: 'container', + summary: `${node.type} · ${count} ${node.type === 'array' ? 'items' : 'keys'}`, + }; +} diff --git a/src/utils/jsonDiff.test.ts b/src/utils/jsonDiff.test.ts index 1e7de6d..3b37b8a 100644 --- a/src/utils/jsonDiff.test.ts +++ b/src/utils/jsonDiff.test.ts @@ -3,10 +3,14 @@ import { charDiff, compareStructured, computeLineDiff, + type DiffBlock, lineStats, + MAX_ARRAY_ALIGN_ITEMS, + MAX_LCS_CELLS, normalize, sortKeysDeep, } from './jsonDiff'; +import { JsonDepthLimitError, MAX_JSON_DEPTH, nestDeeply, runDepthGuarded } from './jsonWalk'; describe('sortKeysDeep', () => { it('recursively sorts object keys and leaves arrays ordered', () => { @@ -53,7 +57,7 @@ describe('normalize', () => { describe('computeLineDiff', () => { it('returns a single equal block for identical texts', () => { const text = '{\n "a": 1\n}'; - const blocks = computeLineDiff(text, text); + const blocks = computeLineDiff(text, text).blocks; expect(blocks).toHaveLength(1); expect(blocks[0]).toMatchObject({ kind: 'equal' }); if (blocks[0].kind === 'equal') { @@ -64,7 +68,7 @@ describe('computeLineDiff', () => { it('detects a modified middle line', () => { const left = '{\n "name": "Alice"\n}'; const right = '{\n "name": "Bob"\n}'; - const blocks = computeLineDiff(left, right); + const blocks = computeLineDiff(left, right).blocks; const change = blocks.find((b) => b.kind === 'change'); expect(change).toBeDefined(); @@ -77,7 +81,7 @@ describe('computeLineDiff', () => { it('detects pure additions', () => { const left = '{\n "a": 1\n}'; const right = '{\n "a": 1,\n "b": 2\n}'; - const blocks = computeLineDiff(left, right); + const blocks = computeLineDiff(left, right).blocks; const change = blocks.find((b) => b.kind === 'change'); expect(change).toBeDefined(); if (change?.kind === 'change') { @@ -111,7 +115,7 @@ describe('charDiff', () => { describe('lineStats', () => { it('counts additions, deletions, and hunks from change blocks', () => { - const blocks = computeLineDiff('a\nb\nc', 'a\nB\nc\nd'); + const blocks = computeLineDiff('a\nb\nc', 'a\nB\nc\nd').blocks; const stats = lineStats(blocks); expect(stats.hunks).toBeGreaterThanOrEqual(1); expect(stats.additions).toBeGreaterThan(0); @@ -119,24 +123,24 @@ describe('lineStats', () => { }); it('returns zeros for identical documents', () => { - const blocks = computeLineDiff('same', 'same'); + const blocks = computeLineDiff('same', 'same').blocks; expect(lineStats(blocks)).toEqual({ additions: 0, deletions: 0, hunks: 0 }); }); }); describe('compareStructured', () => { it('returns no diffs for equal values (including NaN via Object.is)', () => { - expect(compareStructured({ a: 1 }, { a: 1 })).toEqual([]); - expect(compareStructured(null, null)).toEqual([]); + expect(compareStructured({ a: 1 }, { a: 1 }).diffs).toEqual([]); + expect(compareStructured(null, null).diffs).toEqual([]); }); it('reports modified leaves', () => { - const diffs = compareStructured({ a: 1 }, { a: 2 }); + const diffs = compareStructured({ a: 1 }, { a: 2 }).diffs; expect(diffs).toEqual([{ path: '$.a', type: 'modified', leftValue: 1, rightValue: 2 }]); }); it('reports added and removed object keys', () => { - const diffs = compareStructured({ a: 1, b: 2 }, { a: 1, c: 3 }); + const diffs = compareStructured({ a: 1, b: 2 }, { a: 1, c: 3 }).diffs; expect(diffs).toEqual( expect.arrayContaining([ { path: '$.b', type: 'removed', leftValue: 2 }, @@ -149,17 +153,17 @@ describe('compareStructured', () => { it('is key-order independent for objects', () => { const left = { z: 1, a: 2 }; const right = { a: 2, z: 1 }; - expect(compareStructured(left, right)).toEqual([]); + expect(compareStructured(left, right).diffs).toEqual([]); }); it('reports type changes', () => { - expect(compareStructured({ a: 1 }, { a: '1' })).toEqual([ + expect(compareStructured({ a: 1 }, { a: '1' }).diffs).toEqual([ { path: '$.a', type: 'type-changed', leftValue: 1, rightValue: '1' }, ]); }); it('diffs arrays by index', () => { - const diffs = compareStructured([1, 2], [1, 3, 4]); + const diffs = compareStructured([1, 2], [1, 3, 4]).diffs; expect(diffs).toEqual( expect.arrayContaining([ { path: '$[1]', type: 'modified', leftValue: 2, rightValue: 3 }, @@ -169,7 +173,676 @@ describe('compareStructured', () => { }); it('quotes non-identifier keys in paths', () => { - const diffs = compareStructured({ 'a-b': 1 }, { 'a-b': 2 }); + const diffs = compareStructured({ 'a-b': 1 }, { 'a-b': 2 }).diffs; expect(diffs[0]?.path).toBe('$["a-b"]'); }); }); + +/* ------------------------------------------------------------------ * + * Prototype-key safety + * ------------------------------------------------------------------ */ + +describe('prototype-named keys', () => { + it('sortKeysDeep keeps a JSON "__proto__" key as an own property', () => { + const input = JSON.parse('{"z": 1, "__proto__": {"polluted": true}}'); + const out = sortKeysDeep(input); + + expect(Object.hasOwn(out, '__proto__')).toBe(true); + expect(Object.getOwnPropertyDescriptor(out, '__proto__')?.value).toEqual({ polluted: true }); + // The prototype must be untouched — plain assignment would have swapped it. + expect(Object.getPrototypeOf(out)).toBe(Object.prototype); + expect(({} as Record).polluted).toBeUndefined(); + expect(Object.keys(out)).toEqual(['__proto__', 'z']); + }); + + it('normalize keeps a "__proto__" key in the pretty-printed output', () => { + const input = JSON.parse('{"b": 1, "__proto__": {"x": 2}}'); + expect(normalize(input, true)).toBe(`{ + "__proto__": { + "x": 2 + }, + "b": 1 +}`); + }); + + it('compareStructured treats an inherited key as absent, not as an existing one', () => { + // `'constructor' in {}` is true; Object.hasOwn is what this must ask. + expect(compareStructured({}, { constructor: 1 }).diffs).toEqual([ + { path: '$.constructor', type: 'added', rightValue: 1 }, + ]); + expect(compareStructured({ constructor: 1 }, {}).diffs).toEqual([ + { path: '$.constructor', type: 'removed', leftValue: 1 }, + ]); + }); + + it('reports every prototype-named key on either side', () => { + const right = { toString: 'x', hasOwnProperty: 1, valueOf: [2] }; + const diffs = compareStructured({}, right).diffs; + expect(diffs).toEqual( + expect.arrayContaining([ + { path: '$.toString', type: 'added', rightValue: 'x' }, + { path: '$.hasOwnProperty', type: 'added', rightValue: 1 }, + { path: '$.valueOf', type: 'added', rightValue: [2] }, + ]) + ); + expect(diffs).toHaveLength(3); + + const removed = compareStructured(right, {}).diffs; + expect(removed.map((d) => d.type)).toEqual(['removed', 'removed', 'removed']); + }); + + it('handles an own "__proto__" key parsed out of JSON', () => { + const withProto = JSON.parse('{"__proto__": 1}'); + expect(compareStructured({}, withProto).diffs).toEqual([ + { path: '$.__proto__', type: 'added', rightValue: 1 }, + ]); + expect(compareStructured(withProto, JSON.parse('{"__proto__": 2}')).diffs).toEqual([ + { path: '$.__proto__', type: 'modified', leftValue: 1, rightValue: 2 }, + ]); + }); + + it('never reports an inherited function as a diff value', () => { + // CompareWorkspace runs JSON.stringify(...).toLowerCase() over these; a + // function stringifies to undefined and used to throw on the first keypress. + const diffs = compareStructured({}, { constructor: 1, toString: 2, valueOf: 3 }).diffs; + for (const d of diffs) { + expect(typeof d.leftValue).not.toBe('function'); + expect(typeof d.rightValue).not.toBe('function'); + if (d.rightValue !== undefined) expect(JSON.stringify(d.rightValue)).toBeTypeOf('string'); + if (d.leftValue !== undefined) expect(JSON.stringify(d.leftValue)).toBeTypeOf('string'); + } + }); +}); + +/* ------------------------------------------------------------------ * + * Path escaping + * ------------------------------------------------------------------ */ + +describe('structural diff paths', () => { + it('escapes quotes and backslashes in quoted key paths', () => { + expect(compareStructured({ 'a"b': 1 }, { 'a"b': 2 }).diffs[0]?.path).toBe('$["a\\"b"]'); + expect(compareStructured({ 'a\\b': 1 }, { 'a\\b': 2 }).diffs[0]?.path).toBe('$["a\\\\b"]'); + expect(compareStructured({ 'a\nb': 1 }, { 'a\nb': 2 }).diffs[0]?.path).toBe('$["a\\nb"]'); + }); + + it('produces a quoted segment that parses back to the original key', () => { + const key = 'we"ird\\key'; + const path = compareStructured({ [key]: 1 }, { [key]: 2 }).diffs[0]?.path ?? ''; + const quoted = path.slice('$['.length, -']'.length); + expect(JSON.parse(quoted)).toBe(key); + }); + + it('still uses dot notation for identifier-shaped keys', () => { + expect(compareStructured({ $a_1: 1 }, { $a_1: 2 }).diffs[0]?.path).toBe('$.$a_1'); + }); +}); + +/* ------------------------------------------------------------------ * + * Array alignment + * ------------------------------------------------------------------ */ + +describe('compareStructured arrays', () => { + it('reports a front insertion as one addition, not a cascade of modifications', () => { + expect(compareStructured([1, 2, 3], [0, 1, 2, 3]).diffs).toEqual([ + { path: '$[0]', type: 'added', rightValue: 0 }, + ]); + }); + + it('reports a middle insertion as one addition', () => { + expect(compareStructured(['a', 'b'], ['a', 'x', 'b']).diffs).toEqual([ + { path: '$[1]', type: 'added', rightValue: 'x' }, + ]); + }); + + it('reports a front removal as one removal', () => { + expect(compareStructured([0, 1, 2, 3], [1, 2, 3]).diffs).toEqual([ + { path: '$[0]', type: 'removed', leftValue: 0 }, + ]); + }); + + it('aligns arrays of objects across an insertion', () => { + const left = [ + { id: 1, v: 'a' }, + { id: 2, v: 'b' }, + ]; + const right = [ + { id: 0, v: 'z' }, + { id: 1, v: 'a' }, + { id: 2, v: 'b' }, + ]; + expect(compareStructured(left, right).diffs).toEqual([ + { path: '$[0]', type: 'added', rightValue: { id: 0, v: 'z' } }, + ]); + }); + + it('pairs an edited item at its right-hand index after an insertion', () => { + // $[2] is where the edited item lives in the updated document. + const diffs = compareStructured([{ a: 1 }, { a: 2 }], [{ a: 0 }, { a: 1 }, { a: 99 }]).diffs; + expect(diffs).toEqual([ + { path: '$[0]', type: 'added', rightValue: { a: 0 } }, + { path: '$[2].a', type: 'modified', leftValue: 2, rightValue: 99 }, + ]); + }); + + it('reports a removed tail by left index', () => { + expect(compareStructured([1, 2, 3], [1]).diffs).toEqual([ + { path: '$[1]', type: 'removed', leftValue: 2 }, + { path: '$[2]', type: 'removed', leftValue: 3 }, + ]); + }); + + it('walks nested arrays of objects down to the changed leaf', () => { + const left = { rows: [{ cells: [1, 2] }, { cells: [3, 4] }] }; + const right = { rows: [{ cells: [1, 2] }, { cells: [3, 5] }] }; + expect(compareStructured(left, right).diffs).toEqual([ + { path: '$.rows[1].cells[1]', type: 'modified', leftValue: 4, rightValue: 5 }, + ]); + }); + + it('keeps item alignment stable when keys are reordered', () => { + expect(compareStructured([{ a: 1, b: 2 }], [{ b: 2, a: 1 }]).diffs).toEqual([]); + }); + + it('reuses an index across a pair and a leftover removal in the same run', () => { + // Aligned array paths are per-side hints, not unique ids: the pair lands on + // the right index ($[2] = 'X') and the leftover removal keeps its left one + // (also 2). Consumers must not key rows by path alone. + expect(compareStructured(['K', 'A', 'B'], ['N', 'K', 'X']).diffs).toEqual([ + { path: '$[0]', type: 'added', rightValue: 'N' }, + { path: '$[2]', type: 'modified', leftValue: 'A', rightValue: 'X' }, + { path: '$[2]', type: 'removed', leftValue: 'B' }, + ]); + }); + + it('falls back to positional comparison above MAX_ARRAY_ALIGN_ITEMS', () => { + const n = MAX_ARRAY_ALIGN_ITEMS + 1; + const left = Array.from({ length: n }, (_, i) => i + 1); + const right = Array.from({ length: n + 1 }, (_, i) => i); + + const { diffs, positionalArrays } = compareStructured(left, right); + // Positional: every index shifted by the front insertion reads as modified. + expect(diffs).toHaveLength(n + 1); + expect(diffs.filter((d) => d.type === 'modified')).toHaveLength(n); + expect(diffs[diffs.length - 1]).toEqual({ + path: `$[${n}]`, + type: 'added', + rightValue: n, + }); + // ...and the result says so, so the view can disclose it rather than + // presenting 2001 phantom edits as the answer. + expect(positionalArrays).toEqual(['$']); + }); + + it('reports the path of a nested array that degraded, not just the root', () => { + // The signal has to survive the recursion: the walk that gives up is two + // levels below the call that builds the result. + const n = MAX_ARRAY_ALIGN_ITEMS + 1; + const rows = (offset: number) => Array.from({ length: n }, (_, i) => i + offset); + const { diffs, positionalArrays } = compareStructured( + { meta: { name: 'a' }, table: { rows: rows(0) } }, + { meta: { name: 'b' }, table: { rows: rows(1) } } + ); + + expect(positionalArrays).toEqual(['$.table.rows']); + expect(diffs[0]).toEqual({ + path: '$.meta.name', + type: 'modified', + leftValue: 'a', + rightValue: 'b', + }); + }); + + it('degrades when either side alone is over the cap', () => { + // The guard is on both lengths, so a long array compared against a short + // one is positional too — and that is exactly the shape (a truncated list) + // where the answer looks most wrong. + const long = Array.from({ length: MAX_ARRAY_ALIGN_ITEMS + 1 }, (_, i) => i); + expect(compareStructured(long, [0, 1, 2]).positionalArrays).toEqual(['$']); + expect(compareStructured([0, 1, 2], long).positionalArrays).toEqual(['$']); + }); + + it('lists each degraded array once, in the order it was reached', () => { + const big = (offset: number) => + Array.from({ length: MAX_ARRAY_ALIGN_ITEMS + 1 }, (_, i) => i + offset); + const { positionalArrays } = compareStructured( + { a: big(0), b: big(0) }, + { a: big(1), b: big(1) } + ); + expect(positionalArrays).toEqual(['$.a', '$.b']); + }); + + it('reports no degradation for arrays under the cap', () => { + expect(compareStructured([1, 2, 3], [1, 9, 3]).positionalArrays).toEqual([]); + expect(compareStructured({ a: 1 }, { a: 2 }).positionalArrays).toEqual([]); + }); + + it('still aligns arrays of a realistic size', () => { + // The bound has to leave room for ordinary record arrays: at 500 items a + // front insertion must stay one addition, not 501 modifications. The last + // size puts the inserted-into side exactly on the bound, which is inclusive. + for (const n of [100, 500, MAX_ARRAY_ALIGN_ITEMS - 1]) { + const row = (i: number) => ({ id: i, name: `n${i}`, meta: { x: i } }); + const left = Array.from({ length: n }, (_, i) => row(i)); + const right = [row(-1), ...left]; + const { diffs, positionalArrays } = compareStructured(left, right); + expect(diffs, `n=${n}`).toEqual([{ path: '$[0]', type: 'added', rightValue: row(-1) }]); + expect(positionalArrays, `n=${n}`).toEqual([]); + } + }); + + it('does not spread a large sub-result onto the argument stack', () => { + // `diffs.push(...subDiffs)` throws RangeError once one node produces more + // diffs than the engine's argument limit; the walk pushes into a shared + // accumulator instead. 200k leaves under one key is the smallest shape that + // reliably trips the old form. + const size = 200_000; + const left = { rows: Object.fromEntries(Array.from({ length: size }, (_, i) => [i, 1])) }; + const right = { rows: Object.fromEntries(Array.from({ length: size }, (_, i) => [i, 2])) }; + expect(compareStructured(left, right).diffs).toHaveLength(size); + }); +}); + +/* ------------------------------------------------------------------ * + * Structural edge cases + * ------------------------------------------------------------------ */ + +describe('compareStructured edge cases', () => { + it('reports null vs {} and {} vs [] as type changes', () => { + expect(compareStructured(null, {}).diffs).toEqual([ + { path: '$', type: 'type-changed', leftValue: null, rightValue: {} }, + ]); + expect(compareStructured({}, null).diffs).toEqual([ + { path: '$', type: 'type-changed', leftValue: {}, rightValue: null }, + ]); + expect(compareStructured({}, []).diffs).toEqual([ + { path: '$', type: 'type-changed', leftValue: {}, rightValue: [] }, + ]); + expect(compareStructured([], {}).diffs).toEqual([ + { path: '$', type: 'type-changed', leftValue: [], rightValue: {} }, + ]); + }); + + it('finds no diffs between two empty containers of the same kind', () => { + expect(compareStructured({}, {}).diffs).toEqual([]); + expect(compareStructured([], []).diffs).toEqual([]); + }); + + it('distinguishes empty and whitespace-only strings', () => { + expect(compareStructured('', '').diffs).toEqual([]); + expect(compareStructured({ a: '' }, { a: ' ' }).diffs).toEqual([ + { path: '$.a', type: 'modified', leftValue: '', rightValue: ' ' }, + ]); + expect(compareStructured({ a: ' ' }, { a: ' ' }).diffs).toEqual([ + { path: '$.a', type: 'modified', leftValue: ' ', rightValue: ' ' }, + ]); + }); +}); + +/* ------------------------------------------------------------------ * + * Line diff edge cases + * ------------------------------------------------------------------ */ + +describe('computeLineDiff edge cases', () => { + it('numbers the common suffix from each side when the sides differ in length', () => { + // Two lines are inserted on the right, so the shared trailing line is line 3 + // on the left and line 5 on the right. + const blocks = computeLineDiff('head\nkeep\ntail', 'head\nkeep\nnew1\nnew2\ntail').blocks; + const rows = blocks.flatMap((b) => (b.kind === 'equal' ? b.rows : [])); + expect(rows).toEqual([ + { leftNo: 1, rightNo: 1, text: 'head' }, + { leftNo: 2, rightNo: 2, text: 'keep' }, + { leftNo: 3, rightNo: 5, text: 'tail' }, + ]); + expect(lineStats(blocks)).toEqual({ additions: 2, deletions: 0, hunks: 1 }); + }); + + it('numbers the common suffix when the left side is the longer one', () => { + const blocks = computeLineDiff('head\ngone1\ngone2\ntail', 'head\ntail').blocks; + const rows = blocks.flatMap((b) => (b.kind === 'equal' ? b.rows : [])); + expect(rows).toEqual([ + { leftNo: 1, rightNo: 1, text: 'head' }, + { leftNo: 4, rightNo: 2, text: 'tail' }, + ]); + }); + + it('treats two empty documents as one equal row', () => { + expect(computeLineDiff('', '').blocks).toEqual([ + { kind: 'equal', rows: [{ leftNo: 1, rightNo: 1, text: '' }] }, + ]); + expect(lineStats(computeLineDiff('', '').blocks)).toEqual({ + additions: 0, + deletions: 0, + hunks: 0, + }); + }); + + it('diffs an empty document against a non-empty one', () => { + const blocks = computeLineDiff('', 'a\nb').blocks; + expect(lineStats(blocks)).toEqual({ additions: 2, deletions: 1, hunks: 1 }); + const change = blocks.find((b) => b.kind === 'change'); + if (change?.kind === 'change') { + expect(change.removed).toEqual([{ no: 1, text: '' }]); + expect(change.added).toEqual([ + { no: 1, text: 'a' }, + { no: 2, text: 'b' }, + ]); + } + }); + + it('treats whitespace-only lines as content', () => { + expect(lineStats(computeLineDiff(' ', '').blocks)).toEqual({ + additions: 1, + deletions: 1, + hunks: 1, + }); + expect(computeLineDiff(' ', ' ').blocks).toEqual([ + { kind: 'equal', rows: [{ leftNo: 1, rightNo: 1, text: ' ' }] }, + ]); + // Blank lines inside a document are ordinary equal rows. + const blocks = computeLineDiff('a\n\nb', 'a\n\nc').blocks; + expect(lineStats(blocks)).toEqual({ additions: 1, deletions: 1, hunks: 1 }); + }); + + it('falls back to a block replace when the LCS table would exceed MAX_LCS_CELLS', () => { + const n = Math.ceil(Math.sqrt(MAX_LCS_CELLS)) + 1; + expect(n * n).toBeGreaterThan(MAX_LCS_CELLS); + const left = Array.from({ length: n }, (_, i) => `left-${i}`).join('\n'); + const right = Array.from({ length: n }, (_, i) => `right-${i}`).join('\n'); + + const { blocks, lcsFallback } = computeLineDiff(left, right); + // The blocks below are a block replace, not the minimal edit script, so the + // result has to flag itself — the view cannot tell from the blocks alone. + expect(lcsFallback).toBe(true); + expect(blocks).toHaveLength(1); + expect(blocks[0].kind).toBe('change'); + if (blocks[0].kind === 'change') { + expect(blocks[0].removed).toHaveLength(n); + expect(blocks[0].added).toHaveLength(n); + expect(blocks[0].removed[0]).toEqual({ no: 1, text: 'left-0' }); + expect(blocks[0].removed[n - 1]).toEqual({ no: n, text: `left-${n - 1}` }); + expect(blocks[0].added[0]).toEqual({ no: 1, text: 'right-0' }); + expect(blocks[0].added[n - 1]).toEqual({ no: n, text: `right-${n - 1}` }); + } + }); + + it('keeps the untouched prefix and suffix out of the oversized fallback', () => { + const n = Math.ceil(Math.sqrt(MAX_LCS_CELLS)) + 1; + const mid = (tag: string) => Array.from({ length: n }, (_, i) => `${tag}-${i}`).join('\n'); + const { blocks, lcsFallback } = computeLineDiff( + `head\n${mid('left')}\ntail`, + `head\n${mid('right')}\ntail` + ); + expect(lcsFallback).toBe(true); + expect(blocks.map((b) => b.kind)).toEqual(['equal', 'change', 'equal']); + const last = blocks[2]; + if (last.kind === 'equal') { + expect(last.rows).toEqual([{ leftNo: n + 2, rightNo: n + 2, text: 'tail' }]); + } + }); + + it('reports no fallback for a diff the LCS table can hold', () => { + expect(computeLineDiff('a\nb\nc', 'a\nB\nc').lcsFallback).toBe(false); + expect(computeLineDiff('same', 'same').lcsFallback).toBe(false); + expect(computeLineDiff('', '').lcsFallback).toBe(false); + // One side empty is a whole-document add, but it is still the exact answer: + // an empty sequence needs no table, so nothing degraded. + expect(computeLineDiff('', 'a\nb').lcsFallback).toBe(false); + }); +}); + +describe('charDiff prefix/suffix overlap', () => { + it('marks only the appended tail when one line is a strict prefix of the other', () => { + const { left, right } = charDiff('aa', 'aaa'); + expect(left).toEqual([{ text: 'aa', changed: false }]); + expect(right).toEqual([ + { text: 'aa', changed: false }, + { text: 'a', changed: true }, + ]); + }); + + it('marks only the removed tail in the other direction', () => { + const { left, right } = charDiff('aaa', 'aa'); + expect(left).toEqual([ + { text: 'aa', changed: false }, + { text: 'a', changed: true }, + ]); + expect(right).toEqual([{ text: 'aa', changed: false }]); + }); + + it('never double-counts a shared region as both prefix and suffix', () => { + for (const [a, b] of [ + ['aa', 'aaa'], + ['aaa', 'aa'], + ['', 'abc'], + ['abc', ''], + ['ab', 'ba'], + ]) { + const { left, right } = charDiff(a, b); + expect(left.map((s) => s.text).join('')).toBe(a); + expect(right.map((s) => s.text).join('')).toBe(b); + } + }); +}); + +/* ------------------------------------------------------------------ * + * Randomized invariants for the line diff + * ------------------------------------------------------------------ */ + +// Independent reference: plain forward DP, only the LCS *length*, so it shares +// no code path with the engine's backward table. +function lcsLength(a: string[], b: string[]): number { + const dp: number[][] = Array.from({ length: a.length + 1 }, () => + new Array(b.length + 1).fill(0) + ); + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + dp[i][j] = + a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] + 1 : Math.max(dp[i - 1][j], dp[i][j - 1]); + } + } + return dp[a.length][b.length]; +} + +function reconstruct(blocks: DiffBlock[]) { + const left: string[] = []; + const right: string[] = []; + const leftNos: number[] = []; + const rightNos: number[] = []; + for (const block of blocks) { + if (block.kind === 'equal') { + for (const row of block.rows) { + left.push(row.text); + leftNos.push(row.leftNo); + right.push(row.text); + rightNos.push(row.rightNo); + } + } else { + for (const line of block.removed) { + left.push(line.text); + leftNos.push(line.no); + } + for (const line of block.added) { + right.push(line.text); + rightNos.push(line.no); + } + } + } + return { left, right, leftNos, rightNos }; +} + +function makeRng(seed: number) { + let state = seed >>> 0; + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 0x1_0000_0000; + }; +} + +describe('computeLineDiff invariants (randomized)', () => { + it('reconstructs both sides, numbers every line 1..n, and stays minimal', () => { + const rng = makeRng(0xc0ffee); + const alphabet = ['a', 'b', 'c', 'd']; + for (let round = 0; round < 300; round++) { + const gen = () => + Array.from( + { length: 1 + Math.floor(rng() * 8) }, + () => alphabet[Math.floor(rng() * alphabet.length)] + ); + const a = gen(); + const b = gen(); + + const blocks = computeLineDiff(a.join('\n'), b.join('\n')).blocks; + const { left, right, leftNos, rightNos } = reconstruct(blocks); + const label = `${a.join('')} -> ${b.join('')}`; + + expect(left, label).toEqual(a); + expect(right, label).toEqual(b); + expect(leftNos, label).toEqual(a.map((_, i) => i + 1)); + expect(rightNos, label).toEqual(b.map((_, i) => i + 1)); + + const stats = lineStats(blocks); + expect(stats.additions + stats.deletions, label).toBe( + a.length + b.length - 2 * lcsLength(a, b) + ); + expect(stats.hunks, label).toBe(blocks.filter((block) => block.kind === 'change').length); + } + }); + + it('reports zero changes for identical random documents', () => { + const rng = makeRng(7); + for (let round = 0; round < 50; round++) { + const lines = Array.from({ length: 1 + Math.floor(rng() * 12) }, () => + String(Math.floor(rng() * 5)) + ); + const text = lines.join('\n'); + expect(lineStats(computeLineDiff(text, text).blocks)).toEqual({ + additions: 0, + deletions: 0, + hunks: 0, + }); + } + }); +}); + +describe('compareStructured invariants (randomized)', () => { + it('reports diffs exactly when the two values are not deeply equal', () => { + const rng = makeRng(0x5eed); + const leaf = () => { + const pick = Math.floor(rng() * 6); + if (pick === 0) return null; + if (pick === 1) return rng() < 0.5; + if (pick === 2) return Math.floor(rng() * 3); + return ['x', 'y', 'z'][Math.floor(rng() * 3)]; + }; + const build = (depth: number): unknown => { + const pick = rng(); + if (depth <= 0 || pick < 0.45) return leaf(); + if (pick < 0.75) { + return Array.from({ length: Math.floor(rng() * 4) }, () => build(depth - 1)); + } + const obj: Record = {}; + for (const key of ['a', 'b', 'c']) { + if (rng() < 0.6) obj[key] = build(depth - 1); + } + return obj; + }; + + for (let round = 0; round < 400; round++) { + const left = build(2); + const right = build(2); + const diffs = compareStructured(left, right).diffs; + // normalize() sorts keys, so this oracle is key-order independent — the + // same equality compareStructured claims to implement. + const deeplyEqual = normalize(left, true) === normalize(right, true); + const label = `${JSON.stringify(left)} vs ${JSON.stringify(right)}`; + + expect(diffs.length === 0, label).toBe(deeplyEqual); + for (const diff of diffs) { + expect(diff.path.startsWith('$'), label).toBe(true); + expect(['added', 'removed', 'modified', 'type-changed'], label).toContain(diff.type); + } + } + }); + + it('reports nothing for a deep copy of the same value', () => { + const rng = makeRng(11); + for (let round = 0; round < 100; round++) { + const value = { + rows: Array.from({ length: Math.floor(rng() * 6) }, (_, i) => ({ + id: i, + tag: String(Math.floor(rng() * 4)), + })), + }; + expect(compareStructured(value, JSON.parse(JSON.stringify(value))).diffs).toEqual([]); + } + }); +}); + +/* ------------------------------------------------------------------ * + * Depth safety + * ------------------------------------------------------------------ */ + +describe('depth guard', () => { + it('reports a 100,000-level document instead of throwing RangeError', () => { + // `JSON.parse` accepts this shape happily; plain recursion does not survive it. + const deep = nestDeeply(100_000, 'array'); + + expect(runDepthGuarded(() => sortKeysDeep(deep))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + expect(runDepthGuarded(() => normalize(deep, true))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + expect(runDepthGuarded(() => compareStructured(deep, nestDeeply(100_000, 'array')))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + }); + + it('walks a document that sits exactly on the limit', () => { + const atLimit = nestDeeply(MAX_JSON_DEPTH, 'object'); + expect(() => sortKeysDeep(atLimit)).not.toThrow(); + expect(compareStructured(atLimit, atLimit).diffs).toEqual([]); + }); + + it('throws one level past the limit', () => { + const overLimit = nestDeeply(MAX_JSON_DEPTH + 1, 'object'); + expect(() => sortKeysDeep(overLimit)).toThrow(JsonDepthLimitError); + // Two distinct values of the same shape, so the walk actually descends + // rather than reporting a type change at the root and stopping. + expect(() => compareStructured(overLimit, nestDeeply(MAX_JSON_DEPTH + 1, 'object'))).toThrow( + JsonDepthLimitError + ); + }); + + it('counts nesting, not array position', () => { + // `value.map(sortKeysDeep)` would pass each element's index as its depth, so + // a flat array longer than the limit would be rejected as "too deep". + const wide = Array.from({ length: MAX_JSON_DEPTH * 3 }, (_, i) => ({ b: i, a: i })); + expect(() => sortKeysDeep(wide)).not.toThrow(); + expect(Object.keys(sortKeysDeep(wide)[MAX_JSON_DEPTH * 2])).toEqual(['a', 'b']); + }); + + it('aligns long arrays without the item index leaking into the depth', () => { + // compareArrayAligned serialises every item; that map has the same hazard. + const left = Array.from({ length: 1_000 }, (_, i) => ({ id: i })); + const right = left.map((item) => ({ ...item })); + right[900] = { id: -1 }; + const diffs = compareStructured(left, right).diffs; + expect(diffs).toEqual([ + { path: '$[900].id', type: 'modified', leftValue: 900, rightValue: -1 }, + ]); + }); + + it('leaves ordinary documents byte-for-byte unchanged', () => { + const value = { b: [3, { d: 4, c: [5, 6] }], a: { z: 1, y: null } }; + expect(normalize(value, true)).toBe( + JSON.stringify({ a: { y: null, z: 1 }, b: [3, { c: [5, 6], d: 4 }] }, null, 2) + ); + }); +}); diff --git a/src/utils/jsonDiff.ts b/src/utils/jsonDiff.ts index 599f479..468229a 100644 --- a/src/utils/jsonDiff.ts +++ b/src/utils/jsonDiff.ts @@ -9,19 +9,40 @@ // // Everything runs in the browser; no data ever leaves the page. +import { assertJsonDepth, type JsonValue } from './jsonWalk'; + /* ------------------------------------------------------------------ * * Normalization * ------------------------------------------------------------------ */ // Recursively sort object keys so that re-ordered keys don't surface as // textual changes. Arrays keep their order (order is meaningful in JSON arrays). -export function sortKeysDeep(value: any): any { - if (Array.isArray(value)) return value.map(sortKeysDeep); +export function sortKeysDeep(value: any, depth = 0): any { + // `depth` is a real argument now, so the recursion must be wrapped rather than + // passed by reference: `value.map(sortKeysDeep)` would hand each element's + // *index* to `depth` and trip the guard on the 513th member of a flat array. + assertJsonDepth(depth); + if (Array.isArray(value)) return value.map((item) => sortKeysDeep(item, depth + 1)); if (value && typeof value === 'object') { return Object.keys(value) .sort() .reduce>((acc, key) => { - acc[key] = sortKeysDeep(value[key]); + const sorted = sortKeysDeep(value[key], depth + 1); + if (key === '__proto__') { + // `acc.__proto__ = x` hits the setter inherited from Object.prototype: + // it swaps the prototype and silently drops the key. JSON.parse can + // produce a real own "__proto__" key, so define it as a data property + // instead. Only this one key takes the slow path — normalize() runs on + // every keystroke in the compare view. + Object.defineProperty(acc, key, { + value: sorted, + enumerable: true, + writable: true, + configurable: true, + }); + } else { + acc[key] = sorted; + } return acc; }, {}); } @@ -33,65 +54,96 @@ export function normalize(data: any, sortKeys: boolean): string { } /* ------------------------------------------------------------------ * - * Textual line diff + * Sequence diff (shared LCS core) * ------------------------------------------------------------------ */ -type RawOp = - | { type: 'equal'; text: string; leftNo: number; rightNo: number } - | { type: 'remove'; text: string; leftNo: number } - | { type: 'add'; text: string; rightNo: number }; - -export interface EqualRow { - leftNo: number; - rightNo: number; - text: string; -} - -export interface DiffLine { - no: number; - text: string; -} - -export type DiffBlock = - | { kind: 'equal'; rows: EqualRow[] } - | { kind: 'change'; removed: DiffLine[]; added: DiffLine[] }; - -// Cap the LCS table so a pathological input can't freeze the tab. When the -// changed region is larger than this, fall back to a plain block replace. -const MAX_LCS_CELLS = 4_000_000; - -function lcsDiff(a: string[], b: string[], offset: number): RawOp[] { +// Cap the LCS table so a pathological input can't freeze the tab. The table is +// a single flat Uint32Array of (n + 1) * (m + 1) cells — 4 bytes each — so this +// cap is a ~48 MB transient allocation and covers a changed region of roughly +// 3400 x 3400 lines. Beyond it the line diff falls back to a plain block +// replace (delete everything, add everything) and sets `lcsFallback` on its +// result, because that fallback is a wrong-looking answer, not a slower one. +export const MAX_LCS_CELLS = 12_000_000; + +// Aligning array items by content (instead of by position) uses the same table +// plus one stable serialization per item, and runs once per array node rather +// than once per document — so it is bounded by item count, not by cells: two +// 2000-item arrays are a 2000 x 2000 table (16 MB) and measured ~16 ms end to +// end, against ~3 ms for 500 items. Longer arrays fall back to the positional +// walk, which is cheap but re-reports every item after an insertion as +// modified; those arrays are listed in `positionalArrays` so the view can say +// so. Ordinary record arrays must stay under this bound. +export const MAX_ARRAY_ALIGN_ITEMS = 2_000; + +export type EditOp = + | { type: 'equal'; ai: number; bi: number } + | { type: 'remove'; ai: number } + | { type: 'add'; bi: number }; + +// Classic LCS edit script over two string sequences. The dynamic-programming +// table is a flat typed array indexed `i * width + j` rather than an array of +// arrays: one allocation, 4 bytes per cell, and no per-row object headers. +// Callers are responsible for bounding the table size (see the caps above). +function diffSequences(a: string[], b: string[]): EditOp[] { const n = a.length; const m = b.length; - // dp[i][j] = length of the LCS of a[i:] and b[j:] - const dp: number[][] = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0)); + const width = m + 1; + // dp[i * width + j] = length of the LCS of a[i:] and b[j:] + const dp = new Uint32Array((n + 1) * width); for (let i = n - 1; i >= 0; i--) { + const row = i * width; + const nextRow = row + width; for (let j = m - 1; j >= 0; j--) { - dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]); + dp[row + j] = + a[i] === b[j] ? dp[nextRow + j + 1] + 1 : Math.max(dp[nextRow + j], dp[row + j + 1]); } } - const ops: RawOp[] = []; + const ops: EditOp[] = []; let i = 0; let j = 0; while (i < n && j < m) { if (a[i] === b[j]) { - ops.push({ type: 'equal', text: a[i], leftNo: offset + i + 1, rightNo: offset + j + 1 }); + ops.push({ type: 'equal', ai: i, bi: j }); i++; j++; - } else if (dp[i + 1][j] >= dp[i][j + 1]) { - ops.push({ type: 'remove', text: a[i], leftNo: offset + i + 1 }); + } else if (dp[(i + 1) * width + j] >= dp[i * width + j + 1]) { + ops.push({ type: 'remove', ai: i }); i++; } else { - ops.push({ type: 'add', text: b[j], rightNo: offset + j + 1 }); + ops.push({ type: 'add', bi: j }); j++; } } - while (i < n) ops.push({ type: 'remove', text: a[i], leftNo: offset + i++ + 1 }); - while (j < m) ops.push({ type: 'add', text: b[j], rightNo: offset + j++ + 1 }); + while (i < n) ops.push({ type: 'remove', ai: i++ }); + while (j < m) ops.push({ type: 'add', bi: j++ }); return ops; } +/* ------------------------------------------------------------------ * + * Textual line diff + * ------------------------------------------------------------------ */ + +type RawOp = + | { type: 'equal'; text: string; leftNo: number; rightNo: number } + | { type: 'remove'; text: string; leftNo: number } + | { type: 'add'; text: string; rightNo: number }; + +export interface EqualRow { + leftNo: number; + rightNo: number; + text: string; +} + +export interface DiffLine { + no: number; + text: string; +} + +export type DiffBlock = + | { kind: 'equal'; rows: EqualRow[] } + | { kind: 'change'; removed: DiffLine[]; added: DiffLine[] }; + function opsToBlocks(ops: RawOp[]): DiffBlock[] { const blocks: DiffBlock[] = []; let equal: EqualRow[] = []; @@ -127,10 +179,25 @@ function opsToBlocks(ops: RawOp[]): DiffBlock[] { return blocks; } +export interface LineDiffResult { + blocks: DiffBlock[]; + /** + * True when the changed region was too large for the LCS table (see + * {@link MAX_LCS_CELLS}) and was reported as one block replace instead: every + * line of the left region marked removed and every line of the right region + * marked added, including lines that are identical on both sides. + * + * The blocks are still a faithful reconstruction of both documents, but they + * are NOT the minimal edit script — so a caller that shows them to a human has + * to say so, otherwise the view claims a whole file changed when one line did. + */ + lcsFallback: boolean; +} + // Diff two multi-line strings. Trims the common prefix/suffix first (the common // case for JSON where only a handful of lines change) so the expensive LCS only // runs on the genuinely differing middle. -export function computeLineDiff(leftText: string, rightText: string): DiffBlock[] { +export function computeLineDiff(leftText: string, rightText: string): LineDiffResult { const a = leftText.split('\n'); const b = rightText.split('\n'); @@ -152,7 +219,9 @@ export function computeLineDiff(leftText: string, rightText: string): DiffBlock[ const midA = a.slice(start, endA); const midB = b.slice(start, endB); + let lcsFallback = false; if (midA.length * midB.length > MAX_LCS_CELLS) { + lcsFallback = true; for (let i = 0; i < midA.length; i++) { ops.push({ type: 'remove', text: midA[i], leftNo: start + i + 1 }); } @@ -160,7 +229,20 @@ export function computeLineDiff(leftText: string, rightText: string): DiffBlock[ ops.push({ type: 'add', text: midB[j], rightNo: start + j + 1 }); } } else if (midA.length || midB.length) { - ops.push(...lcsDiff(midA, midB, start)); + for (const op of diffSequences(midA, midB)) { + if (op.type === 'equal') { + ops.push({ + type: 'equal', + text: midA[op.ai], + leftNo: start + op.ai + 1, + rightNo: start + op.bi + 1, + }); + } else if (op.type === 'remove') { + ops.push({ type: 'remove', text: midA[op.ai], leftNo: start + op.ai + 1 }); + } else { + ops.push({ type: 'add', text: midB[op.bi], rightNo: start + op.bi + 1 }); + } + } } // Common suffix realigns both sides; numbering picks up after each side's mid. @@ -169,7 +251,7 @@ export function computeLineDiff(leftText: string, rightText: string): DiffBlock[ ops.push({ type: 'equal', text: a[endA + k], leftNo: endA + k + 1, rightNo: endB + k + 1 }); } - return opsToBlocks(ops); + return { blocks: opsToBlocks(ops), lcsFallback }; } /* ------------------------------------------------------------------ * @@ -250,44 +332,194 @@ function valueType(value: any): string { return typeof value; } -// Key-order-independent walk. Objects compare by key, arrays by index, leaves by -// value. Reports the most specific changed paths rather than whole subtrees. -export function compareStructured(left: any, right: any, path = '$'): StructuralDiff[] { - const diffs: StructuralDiff[] = []; +// Key-order-independent serialization used only to decide which array items +// correspond to each other. Values the JSON grammar can't express (undefined, +// functions, symbols) stringify to undefined; collapse those to one sentinel — +// the recursive compare below still reports how the pair actually differs. +const UNSERIALIZABLE = '\0unserializable'; - if (Object.is(left, right)) return diffs; +function stableSerialize(value: any, depth: number): string { + const json = JSON.stringify(sortKeysDeep(value, depth)); + return json === undefined ? UNSERIALIZABLE : json; +} + +export interface StructuralDiffResult { + diffs: StructuralDiff[]; + /** + * Paths of the arrays that were too long to align by content (see + * {@link MAX_ARRAY_ALIGN_ITEMS}) and were therefore compared index by index. + * Distinct paths, in the order they were first reached. + * + * A positional comparison never misses a change, but it reports a shifted + * array badly: prepending one item to a 2,001-item array reports 2,001 diffs + * instead of one addition. A caller showing these diffs to a human has to + * disclose that, or the view is confidently wrong about what changed. + */ + positionalArrays: string[]; +} + +// Accumulator threaded through the walk. Sub-walks push into the same arrays +// instead of returning fresh ones: it keeps the degraded-comparison signal +// flowing up from any depth, and avoids `diffs.push(...subDiffs)`, which spreads +// the whole sub-result onto the argument stack and throws RangeError once a +// single node produces enough diffs. +interface StructuralContext { + diffs: StructuralDiff[]; + positionalArrays: string[]; +} + +// Align two arrays by item content instead of by position, so inserting one +// element at the front reports a single addition rather than re-reporting every +// element after it as modified. +function compareArrayAligned( + left: any[], + right: any[], + path: string, + depth: number, + ctx: StructuralContext +): void { + const diffs = ctx.diffs; + // Wrapped, not passed by reference: `map` supplies an index as the second + // argument, which `stableSerialize` now reads as a depth. + const ops = diffSequences( + left.map((item) => stableSerialize(item, depth + 1)), + right.map((item) => stableSerialize(item, depth + 1)) + ); + + // Removed and added items that land in the same run are far more likely to be + // one edited item than an unrelated delete plus insert, so pair them up and + // recurse. A pair is reported at its RIGHT-hand index so that a "modified" + // path resolves in the updated document; a leftover removal keeps its left + // index, the only side it exists on. + // + // That makes an array path a per-side hint, not a unique id: an unbalanced + // run can reuse one index for a pair and for a leftover removal (see + // ['K','A','B'] vs ['N','K','X'] in the tests, which reports $[2] twice). + // No single-index scheme avoids this once items are aligned rather than + // walked by position, so consumers must not key rows by path alone. + let pendingRemoved: number[] = []; + let pendingAdded: number[] = []; + + const flush = () => { + const paired = Math.min(pendingRemoved.length, pendingAdded.length); + for (let k = 0; k < paired; k++) { + const i = pendingRemoved[k]; + const j = pendingAdded[k]; + compareInto(left[i], right[j], `${path}[${j}]`, depth + 1, ctx); + } + for (let k = paired; k < pendingRemoved.length; k++) { + const i = pendingRemoved[k]; + diffs.push({ path: `${path}[${i}]`, type: 'removed', leftValue: left[i] }); + } + for (let k = paired; k < pendingAdded.length; k++) { + const j = pendingAdded[k]; + diffs.push({ path: `${path}[${j}]`, type: 'added', rightValue: right[j] }); + } + pendingRemoved = []; + pendingAdded = []; + }; + + for (const op of ops) { + if (op.type === 'remove') { + pendingRemoved.push(op.ai); + } else if (op.type === 'add') { + pendingAdded.push(op.bi); + } else { + flush(); + compareInto(left[op.ai], right[op.bi], `${path}[${op.bi}]`, depth + 1, ctx); + } + } + flush(); +} + +/** + * Key-order-independent walk. Objects compare by key, arrays by aligned item + * (or by index once an array is too large to align), leaves by value. Reports + * the most specific changed paths rather than whole subtrees. + * + * Returns the diffs together with the paths of any array that had to be + * compared positionally, so the caller can tell the user the answer is coarser + * than usual instead of presenting a shifted array as thousands of edits. + */ +export function compareStructured( + left: any, + right: any, + path = '$', + depth = 0 +): StructuralDiffResult { + const ctx: StructuralContext = { diffs: [], positionalArrays: [] }; + compareInto(left, right, path, depth, ctx); + return { + diffs: ctx.diffs, + // One array is reached once, but a path is not a unique id (see the note in + // compareArrayAligned), so collapse repeats while keeping first-seen order. + positionalArrays: Array.from(new Set(ctx.positionalArrays)), + }; +} + +function compareInto( + left: any, + right: any, + path: string, + depth: number, + ctx: StructuralContext +): void { + assertJsonDepth(depth); + const diffs = ctx.diffs; + + if (Object.is(left, right)) return; const lt = valueType(left); const rt = valueType(right); if (lt !== rt) { diffs.push({ path, type: 'type-changed', leftValue: left, rightValue: right }); - return diffs; + return; } if (lt !== 'object' && lt !== 'array') { if (left !== right) diffs.push({ path, type: 'modified', leftValue: left, rightValue: right }); - return diffs; + return; } if (lt === 'array') { - const max = Math.max(left.length, right.length); + const leftArr = left as JsonValue[]; + const rightArr = right as JsonValue[]; + if (leftArr.length <= MAX_ARRAY_ALIGN_ITEMS && rightArr.length <= MAX_ARRAY_ALIGN_ITEMS) { + compareArrayAligned(leftArr, rightArr, path, depth, ctx); + return; + } + // Too big to align: compare index by index, and record that this array's + // result is positional so the caller can disclose it. + ctx.positionalArrays.push(path); + const max = Math.max(leftArr.length, rightArr.length); for (let i = 0; i < max; i++) { - if (i >= left.length) - diffs.push({ path: `${path}[${i}]`, type: 'added', rightValue: right[i] }); - else if (i >= right.length) - diffs.push({ path: `${path}[${i}]`, type: 'removed', leftValue: left[i] }); - else diffs.push(...compareStructured(left[i], right[i], `${path}[${i}]`)); + if (i >= leftArr.length) + diffs.push({ path: `${path}[${i}]`, type: 'added', rightValue: rightArr[i] }); + else if (i >= rightArr.length) + diffs.push({ path: `${path}[${i}]`, type: 'removed', leftValue: leftArr[i] }); + else compareInto(leftArr[i], rightArr[i], `${path}[${i}]`, depth + 1, ctx); } - return diffs; + return; } const keys = Array.from(new Set([...Object.keys(left), ...Object.keys(right)])); for (const key of keys) { - const keyPath = /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? `${path}.${key}` : `${path}["${key}"]`; - if (!(key in left)) diffs.push({ path: keyPath, type: 'added', rightValue: right[key] }); - else if (!(key in right)) diffs.push({ path: keyPath, type: 'removed', leftValue: left[key] }); - else diffs.push(...compareStructured(left[key], right[key], keyPath)); + // JSON.stringify supplies the quotes and escapes anything inside them, so a + // key containing " or \ still produces a path that parses back to that key. + const keyPath = /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) + ? `${path}.${key}` + : `${path}[${JSON.stringify(key)}]`; + // Object.hasOwn, not `key in obj`: every object inherits "constructor", + // "toString", "valueOf", … from Object.prototype, and `in` would claim a + // key like that already exists on the other side and then diff against an + // inherited function. + if (!Object.hasOwn(left, key)) { + diffs.push({ path: keyPath, type: 'added', rightValue: right[key] }); + } else if (!Object.hasOwn(right, key)) { + diffs.push({ path: keyPath, type: 'removed', leftValue: left[key] }); + } else { + compareInto(left[key], right[key], keyPath, depth + 1, ctx); + } } - return diffs; } diff --git a/src/utils/jsonDocument.test.ts b/src/utils/jsonDocument.test.ts new file mode 100644 index 0000000..8c51c7c --- /dev/null +++ b/src/utils/jsonDocument.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from 'vitest'; +import { + describeDocumentSize, + documentStatus, + formatBytes, + isBlankDocument, + isParseSuccess, + LARGE_DOCUMENT_CHARS, + MAX_DOCUMENT_CHARS, + measureDocument, + parseJsonStrict, + sortObjectKeysDeep, +} from './jsonDocument'; +import { MAX_JSON_DEPTH, nestDeeply, runDepthGuarded } from './jsonWalk'; + +describe('parseJsonStrict', () => { + it('accepts falsy scalars as valid documents', () => { + // The workbench used to decide validity from the parsed value, so every one + // of these perfectly valid documents was reported as "Invalid JSON format". + expect(parseJsonStrict('0')).toEqual({ isValid: true, data: 0 }); + expect(parseJsonStrict('false')).toEqual({ isValid: true, data: false }); + expect(parseJsonStrict('null')).toEqual({ isValid: true, data: null }); + expect(parseJsonStrict('""')).toEqual({ isValid: true, data: '' }); + }); + + it('parses objects and arrays', () => { + expect(parseJsonStrict('{"a":1}')).toEqual({ isValid: true, data: { a: 1 } }); + expect(parseJsonStrict('[1,2]')).toEqual({ isValid: true, data: [1, 2] }); + }); + + it('treats empty and whitespace-only input as invalid, not as a parse crash', () => { + expect(parseJsonStrict('')).toEqual({ isValid: false, data: null, error: 'Empty input' }); + expect(parseJsonStrict(' \n\t ')).toEqual({ + isValid: false, + data: null, + error: 'Empty input', + }); + }); + + it('reports invalid input with the parser message and does NOT repair it', () => { + // Pins the hot path: repairable-but-invalid input stays invalid here. The + // repair pipeline is reserved for explicit actions (Format / Paste / Open). + const unquotedKeys = parseJsonStrict('{a: 1}'); + expect(unquotedKeys.isValid).toBe(false); + expect(unquotedKeys.data).toBeNull(); + expect(unquotedKeys.error).toBeTruthy(); + + expect(parseJsonStrict('{"a":1,}').isValid).toBe(false); + expect(parseJsonStrict('not json').isValid).toBe(false); + }); +}); + +describe('isParseSuccess', () => { + it('is decided by the error, not by the truthiness of the value', () => { + expect(isParseSuccess({ error: undefined })).toBe(true); + expect(isParseSuccess({ error: 'Unexpected token' })).toBe(false); + }); + + it('treats falsy parsed values as successes', () => { + // Shape mirrors JSONFixer.parseWithFixInfo's result. + for (const data of [0, false, null, '', Number.NaN]) { + expect(isParseSuccess({ data, wasFixed: false } as { data: unknown; error?: string })).toBe( + true + ); + } + }); + + it('treats a failed parse as a failure even though data is null in both cases', () => { + expect( + isParseSuccess({ data: null, wasFixed: false } as { data: unknown; error?: string }) + ).toBe(true); + expect( + isParseSuccess({ data: null, wasFixed: false, error: 'Invalid JSON' } as { + data: unknown; + error?: string; + }) + ).toBe(false); + }); +}); + +describe('documentStatus', () => { + it('reports an empty or whitespace-only document as empty, never invalid', () => { + expect(documentStatus('', false)).toBe('empty'); + expect(documentStatus(' ', false)).toBe('empty'); + expect(documentStatus('\n\t ', true)).toBe('empty'); + }); + + it('reports the valid document "null" as valid', () => { + // The old status bar inferred validity as `json !== null && json !== undefined`, + // which marked the valid document `null` invalid. + expect(documentStatus('null', true)).toBe('valid'); + expect(documentStatus('0', true)).toBe('valid'); + expect(documentStatus('false', true)).toBe('valid'); + expect(documentStatus('""', true)).toBe('valid'); + }); + + it('reports unparsed content as invalid', () => { + expect(documentStatus('{"a":', false)).toBe('invalid'); + }); +}); + +describe('isBlankDocument', () => { + it('detects blank documents', () => { + expect(isBlankDocument('')).toBe(true); + expect(isBlankDocument(' ')).toBe(true); + expect(isBlankDocument('\n\r\t ')).toBe(true); + }); + + it('detects non-blank documents', () => { + expect(isBlankDocument('0')).toBe(false); + expect(isBlankDocument(' {} ')).toBe(false); + }); + + it('agrees with trim() on a large document without allocating a copy', () => { + const big = `${' '.repeat(100_000)}{"a":1}${' '.repeat(100_000)}`; + expect(isBlankDocument(big)).toBe(big.trim() === ''); + expect(isBlankDocument(' '.repeat(100_000))).toBe(true); + }); +}); + +describe('sortObjectKeysDeep', () => { + it('sorts keys A→Z', () => { + expect(Object.keys(sortObjectKeysDeep({ c: 1, a: 2, b: 3 }))).toEqual(['a', 'b', 'c']); + }); + + it('sorts nested objects', () => { + const sorted = sortObjectKeysDeep({ b: { z: 1, y: { d: 1, c: 2 } }, a: 1 }); + expect(JSON.stringify(sorted)).toBe('{"a":1,"b":{"y":{"c":2,"d":1},"z":1}}'); + }); + + it('keeps array order but sorts objects inside arrays', () => { + const sorted = sortObjectKeysDeep({ list: [{ b: 1, a: 2 }, { d: 3, c: 4 }, 'z', 'a'] }); + expect(JSON.stringify(sorted)).toBe('{"list":[{"a":2,"b":1},{"c":4,"d":3},"z","a"]}'); + }); + + it('keeps a member literally named __proto__ instead of dropping it', () => { + // Plain `acc[key] = …` assignment invokes the inherited __proto__ setter: + // the member vanishes from the output and the prototype is replaced by the + // user's data. Verified failing input: {"__proto__":{"polluted":true},"a":2} + // used to sort to {"a":2}. + const input = JSON.parse('{"__proto__": {"polluted": true}, "a": 2}'); + const sorted = sortObjectKeysDeep(input) as Record; + + expect(Object.keys(sorted)).toEqual(['__proto__', 'a']); + expect(Object.getPrototypeOf(sorted)).toBe(Object.prototype); + expect(JSON.stringify(sorted)).toBe('{"__proto__":{"polluted":true},"a":2}'); + expect(({} as Record).polluted).toBeUndefined(); + }); + + it('keeps a nested member named __proto__', () => { + const input = JSON.parse('{"outer": {"z": 1, "__proto__": {"polluted": true}}}'); + expect(JSON.stringify(sortObjectKeysDeep(input))).toBe( + '{"outer":{"__proto__":{"polluted":true},"z":1}}' + ); + }); + + it('keeps other prototype-shadowing member names', () => { + const input = JSON.parse('{"toString": 1, "constructor": 2, "hasOwnProperty": 3, "a": 4}'); + const sorted = sortObjectKeysDeep(input) as Record; + expect(Object.keys(sorted)).toEqual(['a', 'constructor', 'hasOwnProperty', 'toString']); + expect(sorted.constructor).toBe(2); + }); + + it('passes primitives and null through unchanged', () => { + expect(sortObjectKeysDeep(null)).toBeNull(); + expect(sortObjectKeysDeep(0)).toBe(0); + expect(sortObjectKeysDeep(false)).toBe(false); + expect(sortObjectKeysDeep('')).toBe(''); + expect(sortObjectKeysDeep([])).toEqual([]); + expect(sortObjectKeysDeep({})).toEqual({}); + }); + + it('does not mutate its input', () => { + const input = { b: 1, a: { d: 2, c: 3 } }; + const sorted = sortObjectKeysDeep(input); + expect(Object.keys(input)).toEqual(['b', 'a']); + expect(sorted).not.toBe(input); + }); +}); + +describe('measureDocument', () => { + const cases: string[] = [ + '', + '{}', + '{"a":1}\n', + 'line1\nline2\nline3', + '\n\n\n', + '{"name":"café"}', + '{"emoji":"🚀 build"}', + 'a\ud800b', // lone high surrogate + 'a\udc00b', // lone low surrogate + '\ud83d', // trailing lone high surrogate + '日本語テキスト\nsecond line', + ' 
x', + ]; + + it('matches TextEncoder byte length and split() line count exactly', () => { + const encoder = new TextEncoder(); + for (const sample of cases) { + const { bytes, lines } = measureDocument(sample); + expect(bytes, `bytes for ${JSON.stringify(sample)}`).toBe(encoder.encode(sample).length); + expect(lines, `lines for ${JSON.stringify(sample)}`).toBe(sample.split('\n').length); + } + }); + + it('matches on a realistic multi-line document', () => { + const doc = JSON.stringify( + { users: [{ name: 'José', tag: '🚀' }, { name: 'Ann' }], total: 2 }, + null, + 2 + ); + expect(measureDocument(doc)).toEqual({ + bytes: new TextEncoder().encode(doc).length, + lines: doc.split('\n').length, + }); + }); + + it('counts lines as newlines + 1', () => { + expect(measureDocument('a').lines).toBe(1); + expect(measureDocument('a\n').lines).toBe(2); + expect(measureDocument('a\nb').lines).toBe(2); + }); +}); + +describe('describeDocumentSize', () => { + it('buckets by length', () => { + expect(describeDocumentSize(0)).toBe('normal'); + expect(describeDocumentSize(LARGE_DOCUMENT_CHARS)).toBe('normal'); + expect(describeDocumentSize(LARGE_DOCUMENT_CHARS + 1)).toBe('large'); + expect(describeDocumentSize(MAX_DOCUMENT_CHARS)).toBe('large'); + expect(describeDocumentSize(MAX_DOCUMENT_CHARS + 1)).toBe('over-max'); + }); +}); + +describe('formatBytes', () => { + it('scales the unit', () => { + expect(formatBytes(0)).toBe('0 B'); + expect(formatBytes(512)).toBe('512 B'); + expect(formatBytes(2048)).toBe('2.0 KB'); + expect(formatBytes(5 * 1024 * 1024)).toBe('5.00 MB'); + }); +}); + +describe('sortObjectKeysDeep - depth guard', () => { + it('reports a 100,000-level document instead of throwing RangeError', () => { + expect(runDepthGuarded(() => sortObjectKeysDeep(nestDeeply(100_000, 'object')))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + }); + + it('sorts a document that sits exactly on the limit', () => { + expect(runDepthGuarded(() => sortObjectKeysDeep(nestDeeply(MAX_JSON_DEPTH, 'object'))).ok).toBe( + true + ); + }); + + it('counts nesting, not array position', () => { + const wide = Array.from({ length: MAX_JSON_DEPTH * 3 }, (_, i) => ({ b: i, a: i })); + expect(Object.keys(sortObjectKeysDeep(wide)[MAX_JSON_DEPTH * 2])).toEqual(['a', 'b']); + }); + + it('does not change results for ordinary data', () => { + expect(sortObjectKeysDeep({ b: 1, a: { d: 2, c: 3 } })).toEqual({ a: { c: 3, d: 2 }, b: 1 }); + expect(JSON.stringify(sortObjectKeysDeep({ b: 1, a: { d: 2, c: 3 } }))).toBe( + '{"a":{"c":3,"d":2},"b":1}' + ); + }); +}); diff --git a/src/utils/jsonDocument.ts b/src/utils/jsonDocument.ts new file mode 100644 index 0000000..3c4e658 --- /dev/null +++ b/src/utils/jsonDocument.ts @@ -0,0 +1,170 @@ +/** + * Pure helpers for the workbench document: parsing it for validity, measuring + * it, describing its size, and sorting its keys. + * + * These live outside the components so the decisions they encode can be + * unit-tested without a DOM, and so the hot path (every keystroke) is made of + * functions whose cost is obvious at the call site. + */ + +import { assertJsonDepth } from './jsonWalk'; + +/** Above this the editor warns that things may get slow. */ +export const LARGE_DOCUMENT_CHARS = 2 * 1024 * 1024; // ~2MB +/** Hard cap for content arriving from outside the editor (file, drop, clipboard button). */ +export const MAX_DOCUMENT_CHARS = 10 * 1024 * 1024; // ~10MB + +export type DocumentSize = 'normal' | 'large' | 'over-max'; + +/** + * Bucket a document by length. Character count is a deliberate proxy for byte + * size: it is O(1), and the thresholds are advisory. + */ +export function describeDocumentSize(length: number): DocumentSize { + if (length > MAX_DOCUMENT_CHARS) return 'over-max'; + if (length > LARGE_DOCUMENT_CHARS) return 'large'; + return 'normal'; +} + +export interface ParsedDocument { + /** Whether the text is valid JSON. Never infer this from `data`. */ + isValid: boolean; + /** The parsed value when `isValid`, otherwise `null`. */ + data: any; + /** Parser message when invalid. */ + error?: string; +} + +/** + * Strict parse — `JSON.parse` and nothing else. + * + * This is what the live/debounced validation path uses. Running the repair + * pipeline here would run it on every keystroke, since a document being typed + * is invalid for as long as it is unfinished, and repair is super-linear on + * large input. Repair belongs on explicit actions (Format, Paste, Open). + */ +export function parseJsonStrict(text: string): ParsedDocument { + if (isBlankDocument(text)) { + return { isValid: false, data: null, error: 'Empty input' }; + } + try { + return { isValid: true, data: JSON.parse(text) }; + } catch (error) { + return { + isValid: false, + data: null, + error: error instanceof Error ? error.message : 'Invalid JSON', + }; + } +} + +/** + * Did a parse succeed? Decided by the absence of an error, never by the + * truthiness of the value: `0`, `false`, `null` and `""` are all valid JSON + * documents, and testing the value reports every one of them as invalid. + */ +export function isParseSuccess(result: { error?: string }): boolean { + return result.error === undefined; +} + +export type DocumentStatus = 'empty' | 'valid' | 'invalid'; + +/** + * What the status pill should say. Emptiness wins over validity so a blank + * editor reads "Ready" rather than "Invalid JSON". + */ +export function documentStatus(content: string, isValid: boolean): DocumentStatus { + if (isBlankDocument(content)) return 'empty'; + return isValid ? 'valid' : 'invalid'; +} + +/** + * Is the document empty or whitespace-only? Anchored so a non-blank document + * bails at the first character instead of allocating a trimmed copy of what may + * be megabytes of text. + */ +export function isBlankDocument(content: string): boolean { + return content.length === 0 || /^\s*$/.test(content); +} + +/** + * Recursively sort object keys A→Z. Arrays keep their order; their elements are + * sorted in place. + * + * Built with `Object.fromEntries`, which defines each member with + * CreateDataProperty semantics. Plain assignment (`acc[key] = …`) instead + * invokes the inherited `__proto__` *setter* for a member literally named + * `__proto__`: the member is silently dropped from the output and the result's + * prototype is replaced by user-supplied data. + * + * Throws `JsonDepthLimitError` past `MAX_JSON_DEPTH`. Callers run it inside + * `runDepthGuarded` and report that the document is too deeply nested to sort. + */ +export function sortObjectKeysDeep(value: T, depth = 0): T { + assertJsonDepth(depth); + if (Array.isArray(value)) { + return value.map((item) => sortObjectKeysDeep(item, depth + 1)) as unknown as T; + } + if (value !== null && typeof value === 'object') { + const source = value as Record; + return Object.fromEntries( + Object.keys(source) + .sort() + .map((key) => [key, sortObjectKeysDeep(source[key], depth + 1)]) + ) as unknown as T; + } + return value; +} + +export interface DocumentMetrics { + /** UTF-8 length, identical to `new TextEncoder().encode(content).length`. */ + bytes: number; + /** Line count, identical to `content.split('\n').length`. */ + lines: number; +} + +/** + * Measure the document in a single pass, allocating nothing. + * + * The status bar re-measures on every keystroke, so the obvious spelling + * (`new TextEncoder().encode(content).length` plus `content.split('\n').length`) + * allocates a full byte copy of the document *and* an array holding every line, + * per keystroke. This walks the string once instead. + */ +export function measureDocument(content: string): DocumentMetrics { + const len = content.length; + let bytes = 0; + let newlines = 0; + + for (let i = 0; i < len; i++) { + const code = content.charCodeAt(i); + if (code === 10) newlines++; + + if (code < 0x80) { + bytes += 1; + } else if (code < 0x800) { + bytes += 2; + } else if (code >= 0xd800 && code <= 0xdbff) { + // High surrogate: a complete pair is one 4-byte code point; a lone + // surrogate is encoded as U+FFFD (3 bytes), matching TextEncoder. + const next = i + 1 < len ? content.charCodeAt(i + 1) : 0; + if (next >= 0xdc00 && next <= 0xdfff) { + bytes += 4; + i++; + } else { + bytes += 3; + } + } else { + bytes += 3; + } + } + + return { bytes, lines: newlines + 1 }; +} + +/** Human-readable byte size for the status bar. */ +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; +} diff --git a/src/utils/jsonFixer.test.ts b/src/utils/jsonFixer.test.ts index 050faa1..9430754 100644 --- a/src/utils/jsonFixer.test.ts +++ b/src/utils/jsonFixer.test.ts @@ -107,3 +107,419 @@ describe('JSONFixer.parseWithFix', () => { expect(() => JSONFixer.parseWithFix('@@@')).toThrow(); }); }); + +// --------------------------------------------------------------------------- +// Regression suite: the fixer must never rewrite the inside of a string. +// Every case below corrupted user data with the old regex pipeline. +// --------------------------------------------------------------------------- + +/** Turns a valid JSON document into an invalid one by appending a trailing comma. */ +function withTrailingComma(validDoc: string): string { + return `${validDoc.slice(0, -1)},${validDoc.slice(-1)}`; +} + +const HAZARD_VALUES: Record = { + lineComment: 'a value with // inside it', + blockOpen: 'src/*.js', + blockClose: '*/ trailing', + bothCommentMarkers: '/* not a comment */', + url: 'https://example.com/a/b?x=1&y=2#frag', + protocolRelative: '//cdn.example.com/lib.js', + commaThenKey: 'text, key: value', + colonThenWord: ': word,', + brackets: '{ } [ ] {"nested":[1,2]}', + escapedQuote: 'she said "hi" loudly', + backslash: 'C:\\path\\to\\file', + trailingBackslash: 'ends with a backslash \\', + smartQuotes: '\u201Ccurly\u201D and \u2018single\u2019', + pythonWords: 'True False None', + apostrophe: "it's a 'quoted' word", + newlineEscape: 'line1\nline2\ttabbed', + unicodeEscape: 'snowman \u2603 and emoji \u{1f600}', + emptyish: '', +}; + +describe('string literals survive a fix pass byte-identically', () => { + for (const [name, value] of Object.entries(HAZARD_VALUES)) { + it(`preserves a value containing ${name}`, () => { + const valid = JSON.stringify({ v: value }); + const { fixed } = JSONFixer.tryFixJSONWithDetails(withTrailingComma(valid)); + expect(fixed).toBe(valid); + expect(JSON.parse(fixed).v).toBe(value); + }); + + it(`preserves a key containing ${name}`, () => { + const valid = JSON.stringify({ [value]: 1 }); + const { fixed } = JSONFixer.tryFixJSONWithDetails(withTrailingComma(valid)); + expect(fixed).toBe(valid); + expect(JSON.parse(fixed)).toEqual({ [value]: 1 }); + }); + } + + it('preserves every hazard at once in one document', () => { + const valid = JSON.stringify(HAZARD_VALUES); + const result = JSONFixer.parseWithFixInfo(withTrailingComma(valid)); + expect(result.error).toBeUndefined(); + expect(result.wasFixed).toBe(true); + expect(result.data).toEqual(HAZARD_VALUES); + }); + + it('preserves hazards nested in arrays and sub-objects', () => { + const doc = { + list: ['src/*.js', '// not a comment', 'https://x.dev'], + nested: { 'a/*b': { 'c*/d': ['*/', '/*'] } }, + }; + const valid = JSON.stringify(doc); + const result = JSONFixer.parseWithFixInfo(withTrailingComma(valid)); + expect(result.data).toEqual(doc); + }); +}); + +describe('reported corruption reproductions', () => { + it('repairs a trailing comma without eating text between /* and */ in two strings', () => { + const result = JSONFixer.parseWithFixInfo('{"glob": "src/*.js", "other": "*/x", "b": 1,}'); + expect(result.error).toBeUndefined(); + expect(result.wasFixed).toBe(true); + expect(result.data).toEqual({ glob: 'src/*.js', other: '*/x', b: 1 }); + }); + + it('repairs a trailing comma in a document containing a URL', () => { + const result = JSONFixer.parseWithFixInfo('{"url":"https://example.com/a","b":1,}'); + expect(result.error).toBeUndefined(); + expect(result.wasFixed).toBe(true); + expect(result.data).toEqual({ url: 'https://example.com/a', b: 1 }); + }); + + it('does not flatten a nested array that starts with [[', () => { + const result = JSONFixer.parseWithFixInfo('[[1,2],[3,4],]'); + expect(result.error).toBeUndefined(); + expect(result.data).toEqual([ + [1, 2], + [3, 4], + ]); + }); + + it('does not treat a quote inside a string as a string boundary', () => { + const result = JSONFixer.parseWithFixInfo('{"a":"say \\"hi\\"","b":2,}'); + expect(result.data).toEqual({ a: 'say "hi"', b: 2 }); + }); +}); + +describe('valid scalars are returned untouched', () => { + const scalars: Array<[string, unknown]> = [ + ['0', 0], + ['false', false], + ['null', null], + ['""', ''], + ['1e10', 1e10], + ['-1.5e-3', -1.5e-3], + ['"just a string"', 'just a string'], + ['[]', []], + ['{}', {}], + ]; + + it.each(scalars)('parses %s with wasFixed false', (input, expected) => { + const result = JSONFixer.parseWithFixInfo(input); + expect(result).toEqual({ data: expected, wasFixed: false }); + }); + + it('parses -0 without marking it fixed', () => { + const result = JSONFixer.parseWithFixInfo('-0'); + expect(result.wasFixed).toBe(false); + expect(Object.is(result.data, -0)).toBe(true); + }); +}); + +describe('supported repairs', () => { + it('strips a line comment containing quotes and comment markers', () => { + const input = '{\n "a": 1 // don\'t "quote" me /* ok */\n}'; + const result = JSONFixer.parseWithFixInfo(input); + expect(result.data).toEqual({ a: 1 }); + expect(result.fixes).toContain('single-line comments'); + }); + + it('strips a block comment that spans lines', () => { + const result = JSONFixer.parseWithFixInfo('{/* one\ntwo */"a":1}'); + expect(result.data).toEqual({ a: 1 }); + expect(result.fixes).toContain('multi-line comments'); + }); + + it('tolerates an unterminated block comment', () => { + const result = JSONFixer.parseWithFixInfo('{"a":1 /* oops'); + expect(result.data).toEqual({ a: 1 }); + }); + + it('removes trailing commas in arrays and nested objects', () => { + const result = JSONFixer.parseWithFixInfo('{"a":[1,2,],"b":{"c":3,},}'); + expect(result.data).toEqual({ a: [1, 2], b: { c: 3 } }); + expect(result.fixes).toContain('trailing commas'); + }); + + it('removes leading commas', () => { + const result = JSONFixer.parseWithFixInfo('{,"a":1}'); + expect(result.data).toEqual({ a: 1 }); + expect(result.fixes).toContain('leading commas'); + }); + + it('quotes unquoted keys and values', () => { + const result = JSONFixer.parseWithFixInfo('{name: alice, age: 30, ok: true, tags: [a, b]}'); + expect(result.data).toEqual({ name: 'alice', age: 30, ok: true, tags: ['a', 'b'] }); + expect(result.fixes).toContain('unquoted property names'); + expect(result.fixes).toContain('unquoted string values'); + }); + + it('converts single-quoted strings, escaping embedded double quotes', () => { + const result = JSONFixer.parseWithFixInfo("{'msg': 'say \"hi\"', 'it': 'it\\'s'}"); + expect(result.data).toEqual({ msg: 'say "hi"', it: "it's" }); + }); + + it('converts smart double quotes into string delimiters', () => { + const result = JSONFixer.parseWithFixInfo('{\u201Ca\u201D: \u201Cb // c\u201D}'); + expect(result.data).toEqual({ a: 'b // c' }); + expect(result.fixes).toContain('smart double quotes'); + }); + + it('converts smart single quotes into string delimiters', () => { + const result = JSONFixer.parseWithFixInfo('{\u2018a\u2019: \u2018b\u2019}'); + expect(result.data).toEqual({ a: 'b' }); + expect(result.fixes).toContain('smart quotes'); + }); + + it('converts Python literals into JSON literals', () => { + const result = JSONFixer.parseWithFixInfo('{"a": True, "b": False, "c": None, "d": [True]}'); + expect(result.data).toEqual({ a: true, b: false, c: null, d: [true] }); + }); + + it('keeps Python-looking words that are object keys as strings', () => { + const result = JSONFixer.parseWithFixInfo('{True: 1,}'); + expect(result.data).toEqual({ True: 1 }); + }); + + it('adds missing closing brackets of both kinds', () => { + const result = JSONFixer.parseWithFixInfo('{"a":[1,2,{"b":3'); + expect(result.data).toEqual({ a: [1, 2, { b: 3 }] }); + expect(result.fixes?.some((f) => f.includes('closing bracket'))).toBe(true); + }); + + it('does not count brackets that live inside strings', () => { + const result = JSONFixer.parseWithFixInfo('{"a":"{[(", "b":"]}",}'); + expect(result.data).toEqual({ a: '{[(', b: ']}' }); + }); + + it('drops extra closing braces without leaving stray whitespace behind', () => { + expect(JSONFixer.parseWithFixInfo('{"a":1}}').data).toEqual({ a: 1 }); + expect(JSONFixer.parseWithFixInfo('[1,2]]').data).toEqual([1, 2]); + expect(JSONFixer.tryFixJSON('{"a":1}}')).toBe('{"a":1}'); + expect(JSONFixer.tryFixJSON('[1,2]]')).toBe('[1,2]'); + }); + + it('collapses a doubled opening brace at the start', () => { + const result = JSONFixer.parseWithFixInfo('{{"a":1}'); + expect(result.data).toEqual({ a: 1 }); + }); + + it('adds a missing opening brace or bracket', () => { + expect(JSONFixer.parseWithFixInfo('"a":1}').data).toEqual({ a: 1 }); + expect(JSONFixer.parseWithFixInfo('"a":1,"b":2').data).toEqual({ a: 1, b: 2 }); + expect(JSONFixer.parseWithFixInfo('1,2]').data).toEqual([1, 2]); + }); + + it('inserts missing commas between members', () => { + const result = JSONFixer.parseWithFixInfo('{"a":1\n"b":2 "c":[1 2]}'); + expect(result.data).toEqual({ a: 1, b: 2, c: [1, 2] }); + expect(result.fixes).toContain('missing commas between properties'); + }); + + it('closes an unterminated string at end of input', () => { + const result = JSONFixer.parseWithFixInfo('{"a":"abc'); + expect(result.data).toEqual({ a: 'abc' }); + }); + + it('repairs a realistic JS-object-literal blob in one pass', () => { + const input = `{ + // config + name: 'my-app', /* the app */ + entry: "src/*.js", + urls: ['https://a.dev//x', "//cdn.b.dev",], + flags: { debug: True, verbose: False, extra: None, }, + }`; + const result = JSONFixer.parseWithFixInfo(input); + expect(result.error).toBeUndefined(); + expect(result.data).toEqual({ + name: 'my-app', + entry: 'src/*.js', + urls: ['https://a.dev//x', '//cdn.b.dev'], + flags: { debug: true, verbose: false, extra: null }, + }); + }); +}); + +describe('unrepairable input reports an error instead of wrong data', () => { + const broken: Array<[string, string]> = [ + ['garbage punctuation', '@@@'], + ['prose with a brace', 'definitely not { json'], + ['a bare hyphenated word', 'not-json'], + ['a bare word', 'hello'], + ['two top-level numbers', '1 2'], + ['html', 'hi'], + ['three values for one key', '{"a": 1 2 3}'], + ['NaN', '{"a": NaN}'], + ['Infinity', '{"a": Infinity}'], + ['undefined', '{"a": undefined}'], + ['a raw newline inside a string', '{"a": "raw\nnewline in string"}'], + ]; + + it.each(broken)('reports %s as invalid', (_name, input) => { + const result = JSONFixer.parseWithFixInfo(input); + expect(result.data).toBeNull(); + expect(result.wasFixed).toBe(false); + expect(result.error).toBeTruthy(); + }); +}); + +describe('performance', () => { + const PERF_BUDGET_MS = 500; + + function elapsed(fn: () => void): number { + const start = performance.now(); + fn(); + return performance.now() - start; + } + + // Sizing note: the regex this replaced was quadratic in the number of `/*` + // occurrences (measured on this machine: 100KB 82ms, 200KB 328ms, 400KB + // 1309ms, 800KB 5385ms — 4x per doubling). 400KB therefore blows the budget + // on the old implementation while the single-pass scanner needs a few ms. + it('handles ~400KB of unclosed /* inside string values quickly', () => { + const chunk = '/* segment '; + const payload = chunk.repeat(Math.ceil(400_000 / chunk.length)); + const input = `{"a":"${payload}","b":1,}`; + let result: ReturnType | undefined; + const ms = elapsed(() => { + result = JSONFixer.parseWithFixInfo(input); + }); + expect(result?.data).toEqual({ a: payload, b: 1 }); + expect(ms).toBeLessThan(PERF_BUDGET_MS); + }); + + it('handles ~200KB of unclosed /* in code position quickly', () => { + const chunk = '/* note '; + const payload = chunk.repeat(Math.ceil(200_000 / chunk.length)); + const input = `{"a":1, ${payload}`; + let result: ReturnType | undefined; + const ms = elapsed(() => { + result = JSONFixer.parseWithFixInfo(input); + }); + expect(result?.data).toEqual({ a: 1 }); + expect(ms).toBeLessThan(PERF_BUDGET_MS); + }); + + it('handles ~200KB of closed block comments in code position quickly', () => { + const chunk = '/* c */ '; + const count = Math.ceil(200_000 / (chunk.length + 12)); + const parts: string[] = []; + for (let i = 0; i < count; i++) { + parts.push(`${chunk}"k${i}": ${i},`); + } + const input = `{${parts.join('')}}`; + let result: ReturnType | undefined; + const ms = elapsed(() => { + result = JSONFixer.parseWithFixInfo(input); + }); + expect(result?.data?.k0).toBe(0); + expect(ms).toBeLessThan(PERF_BUDGET_MS); + }); + + it('handles a large well-formed document quickly', () => { + const rows = Array.from({ length: 5000 }, (_, i) => ({ + id: i, + url: `https://example.com/item/${i}`, + note: 'src/*.js // keep', + })); + const input = withTrailingComma(JSON.stringify({ rows })); + let result: ReturnType | undefined; + const ms = elapsed(() => { + result = JSONFixer.parseWithFixInfo(input); + }); + expect(result?.data).toEqual({ rows }); + expect(ms).toBeLessThan(PERF_BUDGET_MS); + }); +}); + +describe('JSONFixer.fixMissingBracketsWithDetails', () => { + it('appends the closers that are missing', () => { + const { fixed, fixes } = JSONFixer.fixMissingBracketsWithDetails('{"a":[1,2'); + expect(fixed).toBe('{"a":[1,2]}'); + expect(fixes.length).toBeGreaterThan(0); + }); + + it('leaves balanced input alone', () => { + expect(JSONFixer.fixMissingBracketsWithDetails('{"a":1}')).toEqual({ + fixed: '{"a":1}', + fixes: [], + }); + }); + + it('ignores brackets inside string literals', () => { + expect(JSONFixer.fixMissingBracketsWithDetails('{"a":"}"}')).toEqual({ + fixed: '{"a":"}"}', + fixes: [], + }); + expect(JSONFixer.fixMissingBracketsWithDetails('{"a":"{[("}')).toEqual({ + fixed: '{"a":"{[("}', + fixes: [], + }); + }); + + it('ignores brackets inside comments', () => { + expect(JSONFixer.fixMissingBrackets('{"a":1} // }')).toBe('{"a":1} // }'); + }); + + it('does not add an opening brace because of a colon inside a string', () => { + // The document is `["a\":b"]` — a one-element array whose string contains `":`. + const input = '["a\\":b"]'; + expect(JSONFixer.fixMissingBrackets(input)).toBe(input); + }); +}); + +describe('JSONFixer.fixUnquotedValues', () => { + it('quotes bare identifiers used as values', () => { + const fixes: string[] = []; + expect(JSONFixer.fixUnquotedValues('{"a": bar}', fixes)).toBe('{"a": "bar"}'); + expect(fixes).toContain('unquoted string values'); + }); + + it('leaves JSON literals alone', () => { + const fixes: string[] = []; + expect(JSONFixer.fixUnquotedValues('{"a": true, "b": null}', fixes)).toBe( + '{"a": true, "b": null}' + ); + expect(fixes).toEqual([]); + }); + + it('never rewrites the inside of a string', () => { + const fixes: string[] = []; + const input = '{"a": ": word,", "b": "x, key: value"}'; + expect(JSONFixer.fixUnquotedValues(input, fixes)).toBe(input); + expect(fixes).toEqual([]); + }); + + describe('structure-only garbage is not invented into a value', () => { + // Regression: the stray-closer repair used to prepend an opener even when the + // document had nothing in it, so Format replaced the user's text with a value + // they never wrote — `}{` came back as `{}`, `}` as `{}`, `]` as `[]`. + it.each(['}{', '}', ']', '{}}', '[]]', '}]', ' } '])( + 'rejects %j instead of inventing an empty container', + (input) => { + const result = JSONFixer.parseWithFixInfo(input); + expect(result.error).toBeDefined(); + expect(result.data).toBeNull(); + } + ); + + it('still repairs a missing opener when there IS content to rescue', () => { + expect(JSONFixer.parseWithFixInfo('"a": 1}').data).toEqual({ a: 1 }); + expect(JSONFixer.parseWithFixInfo('1, 2]').data).toEqual([1, 2]); + }); + }); +}); diff --git a/src/utils/jsonFixer.ts b/src/utils/jsonFixer.ts index 78097d5..2f4e473 100644 --- a/src/utils/jsonFixer.ts +++ b/src/utils/jsonFixer.ts @@ -1,3 +1,26 @@ +/** + * Best-effort repair of almost-JSON documents. + * + * The repair is a single left-to-right scan that always knows whether it is + * standing inside a string literal. Every rewrite (comment stripping, comma + * surgery, quote normalisation, bracket balancing) happens ONLY in code + * position; the bytes of a string literal are copied through untouched. That + * is the whole point of the scanner: the previous implementation was a stack + * of global regexes run over the raw text, so a block-comment opener in one + * string and a closer in another looked like a comment and everything between + * them was deleted, and any `https://` URL was eaten as a line comment. + * + * Two consequences worth knowing: + * - The scan is O(n). The old block-comment regex was O(n^2). + * - The repaired text is emitted token by token, so it comes back minified. + * Nothing outside this module's tests consumes the repaired *string*; the + * app consumes `parseWithFixInfo().data` and re-serialises it itself. + * + * When a construct is ambiguous the scanner deliberately copies it through + * unrepaired so `JSON.parse` fails and the caller reports "invalid" — that is + * always better than handing back silently wrong data. + */ + export interface JSONFixResult { data: any; wasFixed: boolean; @@ -5,298 +28,574 @@ export interface JSONFixResult { error?: string; } -export class JSONFixer { - static tryFixJSON(jsonString: string): string { - const result = JSONFixer.tryFixJSONWithDetails(jsonString); - return result.fixed; - } +type Bracket = '{' | '['; - static tryFixJSONWithDetails(jsonString: string): { fixed: string; fixes: string[] } { - let fixed = jsonString.trim(); - const fixes: string[] = []; +/** What the scanner emitted last, in code position. Drives comma insertion. */ +type LastToken = 'none' | 'open' | 'close' | 'colon' | 'value'; - // Remove BOM (Byte Order Mark) if present - if (fixed.charCodeAt(0) === 0xfeff) { - fixed = fixed.slice(1); - fixes.push('BOM character'); - } +const SMART_DOUBLE_QUOTES = '“”'; +const SMART_SINGLE_QUOTES = '‘’'; - // Fix double/multiple opening braces or brackets at the start - if (/^{{+/.test(fixed)) { - fixed = fixed.replace(/^{{+/, '{'); - fixes.push('multiple opening braces'); - } - if (/^\[{2,}/.test(fixed)) { - fixed = fixed.replace(/^\[{2,}/, '['); - fixes.push('multiple opening brackets'); +const JSON_LITERALS = new Set(['true', 'false', 'null']); +const PYTHON_LITERALS: Record = { True: 'true', False: 'false', None: 'null' }; +/** Bare words that must never be turned into strings — they mean "invalid". */ +const NEVER_QUOTED = new Set(['undefined', 'NaN', 'Infinity']); + +function isSpace(ch: string | undefined): boolean { + return ch !== undefined && (ch <= ' ' || ch === '\u00a0' || ch === '\ufeff'); +} + +function isDigit(ch: string): boolean { + return ch >= '0' && ch <= '9'; +} + +function isIdentStart(ch: string): boolean { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_' || ch === '$'; +} + +function isIdentPart(ch: string): boolean { + return isIdentStart(ch) || isDigit(ch); +} + +function isNumberPart(ch: string): boolean { + return isDigit(ch) || ch === '+' || ch === '-' || ch === '.' || ch === 'e' || ch === 'E'; +} + +/** + * Index just past the closing `"` of the JSON string starting at `start`, + * or -1 when the string is never closed. + */ +function endOfDoubleQuoted(src: string, start: number): number { + for (let i = start + 1; i < src.length; i++) { + const ch = src[i]; + if (ch === '\\') { + i++; + continue; } + if (ch === '"') return i + 1; + } + return -1; +} + +/** Index just past a `//` or block comment starting at `start`. */ +function endOfComment(src: string, start: number): number { + if (src[start + 1] === '/') { + let i = start + 2; + while (i < src.length && src[i] !== '\n' && src[i] !== '\r') i++; + return i; + } + const end = src.indexOf('*/', start + 2); + return end === -1 ? src.length : end + 2; +} + +function isCommentStart(src: string, i: number): boolean { + return src[i] === '/' && (src[i + 1] === '/' || src[i + 1] === '*'); +} - // Fix double/multiple closing braces or brackets at the end - if (/}{2,}$/.test(fixed)) { - fixed = fixed.replace(/}{2,}$/, '}'); - fixes.push('multiple closing braces'); +function scanDocument(input: string): { fixed: string; fixes: string[] } { + const fixes = new Set(); + let src = input; + if (src.charCodeAt(0) === 0xfeff) { + src = src.slice(1); + fixes.add('BOM character'); + } + + const n = src.length; + const out: string[] = []; + const stack: Bracket[] = []; + // Positions in `out` of closers that had nothing open to close. + const strayCurly: number[] = []; + const straySquare: number[] = []; + let last: LastToken = 'none'; + let pendingComma = false; + let sawCurlyOpen = false; + let sawSquareOpen = false; + let topLevelColon = false; + let i = 0; + + const nextSignificant = (from: number): string => { + let j = from; + while (j < n && isSpace(src[j])) j++; + return j < n ? src[j] : ''; + }; + + /** Emits whatever separator belongs before the value/opener about to be written. */ + const beforeValue = (): void => { + if (pendingComma) { + pendingComma = false; + if (last === 'open' || last === 'none') { + fixes.add('leading commas'); + } else { + out.push(','); + } + return; } - if (/\]{2,}$/.test(fixed)) { - fixed = fixed.replace(/\]{2,}$/, ']'); - fixes.push('multiple closing brackets'); + if (last !== 'value' && last !== 'close') return; + if (stack.length > 0) { + out.push(','); + fixes.add('missing commas between properties'); + return; } - - // Handle common encoding issues - if (/[\u2018\u2019]/.test(fixed)) { - fixed = fixed.replace(/[\u2018\u2019]/g, "'"); // Smart quotes - fixes.push('smart quotes'); + // Two values side by side at the top level is not a document we can repair + // (`1 2`); keep them apart so they cannot fuse into a single valid token. + out.push(' '); + }; + + /** + * Re-encodes a string that is not delimited by plain double quotes + * (single-quoted or curly-quoted) into a real JSON string. + */ + const readAlienString = (start: number, closers: string): { text: string; next: number } => { + let body = ''; + let j = start + 1; + let terminated = false; + while (j < n) { + const ch = src[j]; + if (ch === '\\') { + const escaped = src[j + 1]; + if (escaped === undefined) { + body += '\\\\'; + j += 1; + break; + } + if (escaped === '"') body += '\\"'; + else if (closers.includes(escaped)) body += escaped; + else body += ch + escaped; + j += 2; + continue; + } + if (closers.includes(ch)) { + terminated = true; + j += 1; + break; + } + body += ch === '"' ? '\\"' : ch; + j += 1; } - if (/[\u201C\u201D]/.test(fixed)) { - fixed = fixed.replace(/[\u201C\u201D]/g, '"'); // Smart double quotes - fixes.push('smart double quotes'); + if (!terminated) fixes.add('unterminated string'); + return { text: `"${body}"`, next: j }; + }; + + while (i < n) { + const ch = src[i]; + + if (isSpace(ch)) { + i++; + continue; } - // Fix single quotes to double quotes (but not inside already quoted strings) - const singleQuoteMatches = fixed.match(/(['"])((?:(?!\1).)*)\1/g); - if (singleQuoteMatches && singleQuoteMatches.some((m) => m[0] === "'")) { - fixed = fixed.replace(/(['"])((?:(?!\1).)*)\1/g, (match, quote, content) => { - return '"' + content + '"'; - }); - fixes.push('single quotes to double quotes'); + if (isCommentStart(src, i)) { + fixes.add(src[i + 1] === '/' ? 'single-line comments' : 'multi-line comments'); + i = endOfComment(src, i); + continue; } - // Remove comments (both single-line and multi-line) - if (/\/\*[\s\S]*?\*\//.test(fixed)) { - fixed = fixed.replace(/\/\*[\s\S]*?\*\//g, ''); // Multi-line comments - fixes.push('multi-line comments'); + // Plain JSON string: copied through byte for byte. + if (ch === '"') { + beforeValue(); + const end = endOfDoubleQuoted(src, i); + if (end === -1) { + out.push(`${src.slice(i)}"`); + fixes.add('unterminated string'); + i = n; + } else { + out.push(src.slice(i, end)); + i = end; + } + last = 'value'; + continue; } - if (/\/\/.*$/m.test(fixed)) { - fixed = fixed.replace(/\/\/.*$/gm, ''); // Single-line comments - fixes.push('single-line comments'); + + if (ch === "'" || SMART_SINGLE_QUOTES.includes(ch)) { + beforeValue(); + const smart = ch !== "'"; + const { text, next } = readAlienString(i, smart ? `${SMART_SINGLE_QUOTES}'` : "'"); + out.push(text); + fixes.add(smart ? 'smart quotes' : 'single quotes to double quotes'); + i = next; + last = 'value'; + continue; } - // Fix missing quotes around property names - if (/([{,]\s*)([a-zA-Z_$][a-zA-Z0-9_$]*)\s*:/.test(fixed)) { - fixed = fixed.replace(/([{,]\s*)([a-zA-Z_$][a-zA-Z0-9_$]*)\s*:/g, '$1"$2":'); - fixes.push('unquoted property names'); + if (SMART_DOUBLE_QUOTES.includes(ch)) { + beforeValue(); + const { text, next } = readAlienString(i, `${SMART_DOUBLE_QUOTES}"`); + out.push(text); + fixes.add('smart double quotes'); + i = next; + last = 'value'; + continue; } - // Fix unquoted string values (JavaScript object literal syntax) - // This is more complex as we need to distinguish between keywords, numbers, and actual unquoted strings - fixed = JSONFixer.fixUnquotedValues(fixed, fixes); + if (ch === '{' || ch === '[') { + beforeValue(); + i++; + // `{{` can never open a valid document (an object key must be a string), + // so a doubled opening brace at the start is a typo worth collapsing. + // `[[` is left alone — it is how every nested array legitimately starts. + if (ch === '{' && out.length === 0) { + let j = i; + while (j < n && isSpace(src[j])) j++; + while (src[j] === '{') { + fixes.add('multiple opening braces'); + j++; + while (j < n && isSpace(src[j])) j++; + } + i = j; + } + out.push(ch); + stack.push(ch); + if (ch === '{') sawCurlyOpen = true; + else sawSquareOpen = true; + last = 'open'; + continue; + } - // Try to fix missing brackets FIRST (before removing trailing commas) - const bracketResult = JSONFixer.fixMissingBracketsWithDetails(fixed); - if (bracketResult.fixes.length > 0) { - fixed = bracketResult.fixed; - fixes.push(...bracketResult.fixes); + if (ch === '}' || ch === ']') { + if (pendingComma) { + pendingComma = false; + fixes.add('trailing commas'); + } + const opener: Bracket = ch === '}' ? '{' : '['; + if (stack.length === 0) { + (ch === '}' ? strayCurly : straySquare).push(out.length); + } else if (stack[stack.length - 1] === opener) { + stack.pop(); + } + // A closer that does not match what is open is emitted as-is: guessing + // which container the author meant to close would invent structure. + out.push(ch); + last = 'close'; + i++; + continue; } - // Remove trailing commas before closing brackets/braces - if (/,\s*[}\]]/.test(fixed)) { - fixed = fixed.replace(/,\s*([}\]])/g, '$1'); - fixes.push('trailing commas'); + if (ch === ',') { + if (pendingComma) fixes.add('multiple consecutive commas'); + pendingComma = true; + i++; + continue; } - // Remove leading commas after opening brackets/braces - if (/[{[]\s*,/.test(fixed)) { - fixed = fixed.replace(/([{[])\s*,/g, '$1'); - fixes.push('leading commas'); + if (ch === ':') { + if (pendingComma) { + pendingComma = false; + out.push(','); + } + if (stack.length === 0) topLevelColon = true; + out.push(':'); + last = 'colon'; + i++; + continue; } - // Fix multiple consecutive commas - if (/,\s*,+/.test(fixed)) { - fixed = fixed.replace(/,\s*,+/g, ','); - fixes.push('multiple consecutive commas'); + if (isDigit(ch) || ch === '-' || ch === '+' || ch === '.') { + beforeValue(); + let j = i; + while (j < n && isNumberPart(src[j])) j++; + out.push(src.slice(i, j)); + i = j; + last = 'value'; + continue; } - // Fix missing commas between properties (but not before closing brackets) - if (/(["\d\]}\w])\s*\n\s*"([^"]+)":/.test(fixed) || /(["\d\]}\w])\s*"([^"]+)":/.test(fixed)) { - fixed = fixed.replace(/(["\d\]}\w])\s*\n\s*"([^"]+)":/g, '$1,\n"$2":'); - fixed = fixed.replace(/(["\d\]}\w])\s*"([^"]+)":/g, '$1,"$2":'); - fixes.push('missing commas between properties'); + if (isIdentStart(ch)) { + let j = i; + while (j < n && (isIdentPart(src[j]) || (src[j] === '.' && isIdentStart(src[j + 1] ?? '')))) { + j++; + } + const word = src.slice(i, j); + const after = nextSignificant(j); + // Only repair a bare word sitting where a key or value belongs. Prose + // (`hello world`, `Note: see below`) must stay unrepairable. + const inSlot = pendingComma || last === 'open' || last === 'colon'; + i = j; + beforeValue(); + last = 'value'; + + if (inSlot && after === ':') { + out.push(JSON.stringify(word)); + fixes.add('unquoted property names'); + continue; + } + if (JSON_LITERALS.has(word)) { + out.push(word); + continue; + } + const python = PYTHON_LITERALS[word]; + if (python !== undefined) { + out.push(python); + fixes.add('Python literals'); + continue; + } + const closesValue = after === ',' || after === '}' || after === ']' || after === ''; + if (inSlot && closesValue && !NEVER_QUOTED.has(word)) { + out.push(JSON.stringify(word)); + fixes.add('unquoted string values'); + continue; + } + // NaN / Infinity / undefined / anything else: copy it through so the + // document fails to parse instead of becoming the string "NaN". + out.push(word); + continue; } - return { fixed, fixes }; + // Unknown character in code position — keep it so the parse fails loudly. + beforeValue(); + out.push(ch); + last = 'value'; + i++; } - static fixMissingBrackets(str: string): string { - const result = JSONFixer.fixMissingBracketsWithDetails(str); - return result.fixed; - } + if (pendingComma) fixes.add('trailing commas'); - static fixMissingBracketsWithDetails(str: string): { fixed: string; fixes: string[] } { - const fixes: string[] = []; - // First, check if the string has any bracket-like structure - const hasCurlyStart = str.includes('{'); - const hasCurlyEnd = str.includes('}'); - const hasSquareStart = str.includes('['); - const hasSquareEnd = str.includes(']'); - - // If it looks like it should be an object but is missing opening bracket - if (!hasCurlyStart && (str.includes('":') || hasCurlyEnd)) { - str = '{' + str; - fixes.push('opening brace {'); + while (stack.length > 0) { + const opener = stack.pop(); + if (opener === '{') { + out.push('}'); + fixes.add('closing brace }'); + } else { + out.push(']'); + fixes.add('closing bracket ]'); } + } - // If it looks like it should be an array but is missing opening bracket - if (!hasSquareStart && hasSquareEnd && !str.includes('":')) { - str = '[' + str; - fixes.push('opening bracket ['); + // Every repair below invents a container, which is only honest when there is + // something to put in it: `"a":1}` really is a missing `{`. A document that is + // nothing but structure (`}`, `]`, `}{`, `{}}`) has no content to rescue, so + // "repairing" it to `{}` would hand back a value the user never wrote — and + // Format writes that straight into their editor. Erroring is the honest answer. + const hasValueContent = out.some((piece) => /[^\s{}[\],:]/.test(piece)); + + // Unmatched closers mean either a missing opener or a duplicated closer. + if (hasValueContent && strayCurly.length > 0) { + if (sawCurlyOpen) { + for (const idx of strayCurly) out[idx] = ''; + fixes.add('multiple closing braces'); + } else { + out.unshift('{'.repeat(strayCurly.length)); + fixes.add('opening brace {'); } + } + if (hasValueContent && straySquare.length > 0) { + if (sawSquareOpen) { + for (const idx of straySquare) out[idx] = ''; + fixes.add('multiple closing brackets'); + } else { + out.unshift('['.repeat(straySquare.length)); + fixes.add('opening bracket ['); + } + } - // Count brackets - let openCurly = 0; - let openSquare = 0; - let inString = false; - let escapeNext = false; - - for (let i = 0; i < str.length; i++) { - const char = str[i]; + // `"a":1, "b":2` — members pasted without the object around them. + if (topLevelColon && !sawCurlyOpen && strayCurly.length === 0) { + out.unshift('{'); + out.push('}'); + fixes.add('opening brace {'); + fixes.add('closing brace }'); + } - if (escapeNext) { - escapeNext = false; - continue; - } + return { fixed: out.join(''), fixes: [...fixes] }; +} - if (char === '\\') { - escapeNext = true; - continue; - } +function tryFixJSONWithDetails(jsonString: string): { fixed: string; fixes: string[] } { + return scanDocument(jsonString); +} - if (char === '"' && !inString) { - inString = true; - } else if (char === '"' && inString) { - inString = false; - } +function tryFixJSON(jsonString: string): string { + return scanDocument(jsonString).fixed; +} - if (!inString) { - if (char === '{') openCurly++; - else if (char === '}') openCurly--; - else if (char === '[') openSquare++; - else if (char === ']') openSquare--; - } +/** + * Balances brackets without touching anything else, preserving the original + * formatting. String- and comment-aware, so braces inside `"{"` or `// {` do + * not count. + */ +function fixMissingBracketsWithDetails(str: string): { fixed: string; fixes: string[] } { + const fixes: string[] = []; + const stack: Bracket[] = []; + let extraCurly = 0; + let extraSquare = 0; + let sawCurlyOpen = false; + let sawSquareOpen = false; + let topLevelColon = false; + + for (let i = 0; i < str.length; i++) { + const ch = str[i]; + if (ch === '"') { + const end = endOfDoubleQuoted(str, i); + i = (end === -1 ? str.length : end) - 1; + continue; + } + if (isCommentStart(str, i)) { + i = endOfComment(str, i) - 1; + continue; } + if (ch === '{' || ch === '[') { + stack.push(ch); + if (ch === '{') sawCurlyOpen = true; + else sawSquareOpen = true; + continue; + } + if (ch === '}' || ch === ']') { + const opener: Bracket = ch === '}' ? '{' : '['; + if (stack[stack.length - 1] === opener) stack.pop(); + else if (stack.length === 0 && ch === '}') extraCurly++; + else if (stack.length === 0) extraSquare++; + continue; + } + if (ch === ':' && stack.length === 0) topLevelColon = true; + } - // Add missing closing brackets - let result = str; + let fixed = str; + const needsWrap = topLevelColon && !sawCurlyOpen && extraCurly === 0; - // Trim any trailing whitespace and commas before adding brackets - result = result.replace(/[\s,]+$/, ''); + if (extraCurly > 0 && !sawCurlyOpen) { + fixed = '{'.repeat(extraCurly) + fixed; + fixes.push(`${extraCurly} opening brace${extraCurly > 1 ? 's' : ''} {`); + } + if (extraSquare > 0 && !sawSquareOpen) { + fixed = '['.repeat(extraSquare) + fixed; + fixes.push(`${extraSquare} opening bracket${extraSquare > 1 ? 's' : ''} [`); + } + if (needsWrap) { + fixed = `{${fixed}`; + fixes.push('opening brace {'); + } - // Add missing closing curly braces - while (openCurly > 0) { - result += '}'; - openCurly--; + if (stack.length > 0 || needsWrap) { + fixed = fixed.replace(/[\s,]+$/, ''); + while (stack.length > 0) { + const opener = stack.pop(); + if (opener === '{') { + fixed += '}'; + fixes.push('closing brace }'); + } else { + fixed += ']'; + fixes.push('closing bracket ]'); + } + } + if (needsWrap) { + fixed += '}'; fixes.push('closing brace }'); } + } - // Add missing closing square brackets - while (openSquare > 0) { - result += ']'; - openSquare--; - fixes.push('closing bracket ]'); - } + return { fixed, fixes }; +} - // Add missing opening brackets if we have more closing than opening - if (openCurly < 0) { - result = '{'.repeat(-openCurly) + result; - fixes.push(`${-openCurly} opening brace${-openCurly > 1 ? 's' : ''} {`); - } +function fixMissingBrackets(str: string): string { + return fixMissingBracketsWithDetails(str).fixed; +} - if (openSquare < 0) { - result = '['.repeat(-openSquare) + result; - fixes.push(`${-openSquare} opening bracket${-openSquare > 1 ? 's' : ''} [`); +/** + * Quotes bare identifiers used as values (`{"a": bar}` -> `{"a": "bar"}`), + * leaving strings, comments and JSON literals alone. Formatting is preserved. + */ +function fixUnquotedValues(str: string, fixes: string[]): string { + const n = str.length; + let out = ''; + let quotedAny = false; + let i = 0; + + while (i < n) { + const ch = str[i]; + if (ch === '"') { + const end = endOfDoubleQuoted(str, i); + const stop = end === -1 ? n : end; + out += str.slice(i, stop); + i = stop; + continue; + } + if (isCommentStart(str, i)) { + const stop = endOfComment(str, i); + out += str.slice(i, stop); + i = stop; + continue; } - return { fixed: result, fixes }; - } + out += ch; + i++; + if (ch !== ':' && ch !== '[' && ch !== ',') continue; - static fixUnquotedValues(str: string, fixes: string[]): string { - let fixed = str; - let hasUnquotedValues = false; - - // More comprehensive pattern to match unquoted values - // This handles: - // 1. Property:value pairs - // 2. Array elements - // 3. Values before commas, brackets, or braces - - // First, fix unquoted values after colons (object properties) - fixed = fixed.replace( - /:\s*([a-zA-Z_$][a-zA-Z0-9_$]*(?:\.[a-zA-Z_$][a-zA-Z0-9_$]*)*)(?=\s*[,\]}])/g, - (match, value) => { - // Skip JSON literals - if (value === 'true' || value === 'false' || value === 'null' || value === 'undefined') { - return match; - } - hasUnquotedValues = true; - return `: "${value}"`; - } - ); - - // Fix unquoted values in arrays - fixed = fixed.replace( - /(\[|,)\s*([a-zA-Z_$][a-zA-Z0-9_$]*(?:\.[a-zA-Z_$][a-zA-Z0-9_$]*)*)(?=\s*[,\]])/g, - (match, prefix, value) => { - // Skip JSON literals - if (value === 'true' || value === 'false' || value === 'null' || value === 'undefined') { - return match; - } - hasUnquotedValues = true; - return `${prefix} "${value}"`; - } - ); + let start = i; + while (start < n && isSpace(str[start])) start++; + if (start >= n || !isIdentStart(str[start])) continue; - if (hasUnquotedValues) { - fixes.push('unquoted string values'); + let end = start; + while ( + end < n && + (isIdentPart(str[end]) || (str[end] === '.' && isIdentStart(str[end + 1] ?? ''))) + ) { + end++; } + const word = str.slice(start, end); - return fixed; + let after = end; + while (after < n && isSpace(str[after])) after++; + const delimiter = after < n ? str[after] : ''; + if (delimiter !== ',' && delimiter !== ']' && delimiter !== '}') continue; + if (JSON_LITERALS.has(word) || NEVER_QUOTED.has(word)) continue; + + out += str.slice(i, start) + JSON.stringify(word); + i = end; + quotedAny = true; } - static parseWithFix(jsonString: string): any { + if (quotedAny) fixes.push('unquoted string values'); + return out; +} + +function parseWithFix(jsonString: string): any { + try { + return JSON.parse(jsonString); + } catch (firstError) { try { - // First attempt: parse as-is - return JSON.parse(jsonString); - } catch (firstError) { - try { - // Second attempt: try to fix common issues - const fixed = JSONFixer.tryFixJSON(jsonString); - return JSON.parse(fixed); - } catch (secondError) { - // If both attempts fail, throw the original error - throw firstError; - } + return JSON.parse(tryFixJSON(jsonString)); + } catch { + // The original error describes the user's document; the error from the + // repaired text describes our guess at it, which is less useful. + throw firstError; } } +} + +function parseWithFixInfo(jsonString: string): JSONFixResult { + if (!jsonString || jsonString.trim() === '') { + return { + data: null, + wasFixed: false, + error: 'Empty input', + }; + } - static parseWithFixInfo(jsonString: string): JSONFixResult { - if (!jsonString || jsonString.trim() === '') { + try { + return { data: JSON.parse(jsonString), wasFixed: false }; + } catch (firstError) { + try { + const fixResult = tryFixJSONWithDetails(jsonString); + const data = JSON.parse(fixResult.fixed); + return { + data, + wasFixed: true, + fixes: fixResult.fixes, + }; + } catch { return { data: null, wasFixed: false, - error: 'Empty input', + error: firstError instanceof Error ? firstError.message : 'Invalid JSON', }; } - - try { - // First attempt: parse as-is - const data = JSON.parse(jsonString); - return { data, wasFixed: false }; - } catch (firstError) { - try { - // Second attempt: try to fix common issues - const fixResult = JSONFixer.tryFixJSONWithDetails(jsonString); - const data = JSON.parse(fixResult.fixed); - return { - data, - wasFixed: true, - fixes: fixResult.fixes, - }; - } catch (secondError) { - // Return with error instead of throwing - return { - data: null, - wasFixed: false, - error: firstError instanceof Error ? firstError.message : 'Invalid JSON', - }; - } - } } } + +export const JSONFixer = { + tryFixJSON, + tryFixJSONWithDetails, + fixMissingBrackets, + fixMissingBracketsWithDetails, + fixUnquotedValues, + parseWithFix, + parseWithFixInfo, +}; diff --git a/src/utils/jsonLossy.test.ts b/src/utils/jsonLossy.test.ts new file mode 100644 index 0000000..45cbca8 --- /dev/null +++ b/src/utils/jsonLossy.test.ts @@ -0,0 +1,501 @@ +import { describe, expect, it } from 'vitest'; +import { + classifyNumberLiteral, + DEFAULT_LOSSY_CHANGE_LIMIT, + describeLossyScan, + type ExactDecimal, + exactDecimal, + LOSSY_KIND_TEXT, + type LossyChangeKind, + sameExactValue, + scanLossyRewrites, +} from './jsonLossy'; + +/** Kinds found in a document, in first-seen order. */ +const kindsOf = (source: string): LossyChangeKind[] => scanLossyRewrites(source).kinds; + +/** The English rendering of a scan, as the toast would show it. */ +const describe_ = (source: string): string | null => + describeLossyScan(scanLossyRewrites(source), (_key, fallback) => fallback); + +/** + * The exact integer `JSON.stringify` emits for an integer literal, as a BigInt. + * + * Deliberately hand-rolled here rather than imported: an oracle that shares code + * with the thing it checks proves nothing. `JSON.stringify` switches to + * exponential notation at 1e21, so `1e+23` is expanded back to digits. + */ +function emittedInteger(literal: string): bigint { + const emitted = JSON.stringify(Number(literal)); + const [mantissa, exponent] = emitted.split('e'); + if (exponent === undefined) return BigInt(mantissa); + const [whole, fraction = ''] = mantissa.split('.'); + return BigInt(whole + fraction + '0'.repeat(Number(exponent) - fraction.length)); +} + +describe('exactDecimal', () => { + it('normalises equal values to identical representations', () => { + expect(exactDecimal('1')).toEqual(exactDecimal('1.0')); + expect(exactDecimal('1')).toEqual(exactDecimal('1.000')); + expect(exactDecimal('1250')).toEqual(exactDecimal('1.25e3')); + expect(exactDecimal('1250')).toEqual(exactDecimal('12500e-1')); + expect(exactDecimal('0')).toEqual(exactDecimal('0.000')); + }); + + it('keeps the significand free of leading and trailing zeros', () => { + expect(exactDecimal('12300')).toEqual({ negative: false, digits: '123', exponent: 2 }); + expect(exactDecimal('0.00450')).toEqual({ negative: false, digits: '45', exponent: -4 }); + expect(exactDecimal('-7')).toEqual({ negative: true, digits: '7', exponent: 0 }); + }); + + it('represents zero as an empty significand, keeping its sign', () => { + expect(exactDecimal('0')?.digits).toBe(''); + expect(exactDecimal('0')?.negative).toBe(false); + expect(exactDecimal('-0')?.digits).toBe(''); + expect(exactDecimal('-0')?.negative).toBe(true); + expect(exactDecimal('-0.00e5')?.negative).toBe(true); + }); + + it('refuses anything that is not a JSON number', () => { + for (const text of [ + '+1', + '.5', + '1.', + '01', + '0x1f', + '1.2.3', + '1e', + 'Infinity', + 'NaN', + '-', + '', + '1_000', + ]) { + expect(exactDecimal(text), text).toBeNull(); + } + }); + + it('survives exponents no double could hold', () => { + expect(exactDecimal('1e999')).toEqual({ negative: false, digits: '1', exponent: 999 }); + expect(exactDecimal('1e-999')).toEqual({ negative: false, digits: '1', exponent: -999 }); + }); +}); + +describe('sameExactValue', () => { + /** `exactDecimal`, failing the test rather than the type checker on `null`. */ + const decimal = (literal: string): ExactDecimal => { + const value = exactDecimal(literal); + if (value === null) throw new Error(`not a JSON number: ${literal}`); + return value; + }; + + it('treats minus zero and zero as different values', () => { + expect(sameExactValue(decimal('-0'), decimal('0'))).toBe(false); + expect(sameExactValue(decimal('0'), decimal('0'))).toBe(true); + expect(sameExactValue(decimal('-0'), decimal('-0'))).toBe(true); + }); + + it('compares by value, not by spelling', () => { + expect(sameExactValue(decimal('1'), decimal('1.0'))).toBe(true); + expect(sameExactValue(decimal('1'), decimal('10e-1'))).toBe(true); + expect(sameExactValue(decimal('1'), decimal('2'))).toBe(false); + expect(sameExactValue(decimal('1'), decimal('1e1'))).toBe(false); + }); +}); + +describe('classifyNumberLiteral', () => { + it('reports every verified way the round trip rewrites a number', () => { + expect(classifyNumberLiteral('12345678901234567890')).toEqual({ + kind: 'unsafe-integer', + becomes: '12345678901234567000', + }); + expect(classifyNumberLiteral('9007199254740993')).toEqual({ + kind: 'unsafe-integer', + becomes: '9007199254740992', + }); + expect(classifyNumberLiteral('1e999')).toEqual({ kind: 'overflow', becomes: 'null' }); + expect(classifyNumberLiteral('-1e999')).toEqual({ kind: 'overflow', becomes: 'null' }); + expect(classifyNumberLiteral('-0')).toEqual({ kind: 'negative-zero', becomes: '0' }); + expect(classifyNumberLiteral('0.1234567890123456789')).toEqual({ + kind: 'precision', + becomes: '0.12345678901234568', + }); + // Underflow is a rounding, so it is reported as one. + expect(classifyNumberLiteral('1e-999')).toEqual({ kind: 'precision', becomes: '0' }); + }); + + it('stays silent about re-spellings that preserve the value', () => { + // `1.0` and `1` are the same JSON number; the output text differs, the data + // does not, and a warning here would train the user to ignore all of them. + expect(classifyNumberLiteral('1.0')).toBeNull(); + expect(classifyNumberLiteral('1.000')).toBeNull(); + expect(classifyNumberLiteral('1e2')).toBeNull(); + expect(classifyNumberLiteral('-0.0e3')).not.toBeNull(); // still minus zero + }); + + it('stays silent about big integers that round-trip exactly', () => { + // Past MAX_SAFE_INTEGER but exactly representable, so nothing is lost. + expect(Number.isSafeInteger(1e19)).toBe(false); + expect(classifyNumberLiteral('10000000000000000000')).toBeNull(); + expect(classifyNumberLiteral('18014398509481984')).toBeNull(); // 2^54 + }); + + it('stays silent when the stored double is inexact but the text survives', () => { + // The sharpest case for the criterion this module uses. The double behind + // 92640008104508060 is not that integer, but it is the *nearest* one, so + // JSON.stringify emits the shortest text that round-trips — the same digits + // the user typed. Format changes nothing, so there is nothing to warn about. + const literal = '92640008104508060'; + expect(Number.isSafeInteger(Number(literal))).toBe(false); + expect(JSON.stringify(JSON.parse(literal))).toBe(literal); + expect(classifyNumberLiteral(literal)).toBeNull(); + }); + + it('makes no claim about text that is not a JSON number', () => { + for (const text of ['+1', '.5', '1.', '01', '0x1f', '1.2.3', '-', 'e5']) { + expect(classifyNumberLiteral(text), text).toBeNull(); + } + }); + + it('agrees with a BigInt oracle on every integer literal', () => { + // Independent check. The question Format asks is whether the *text* changes + // value, so the oracle compares the literal the user wrote with the literal + // `JSON.stringify` emits, both as exact BigInts — no reuse of the decimal + // machinery under test, and no doubles in the comparison. + const literals = [ + '0', + '1', + '-1', + '9007199254740991', + '-9007199254740991', + '9007199254740992', + '9007199254740993', + '-9007199254740993', + '10000000000000000000', + '12345678901234567890', + '18014398509481984', + '18014398509481985', + '99999999999999999999999', + '123456789012345678901234567890', + ]; + + for (const literal of literals) { + const verdict = classifyNumberLiteral(literal); + if (BigInt(literal) === emittedInteger(literal)) { + expect(verdict, literal).toBeNull(); + } else { + expect(verdict?.kind, literal).toBe('unsafe-integer'); + } + } + }); + + it('agrees with a BigInt oracle on randomly generated big integers', () => { + let seed = 0x5eed_1234; + const nextDigit = () => { + // xorshift, so the case list is wide but the test stays deterministic. + seed ^= seed << 13; + seed ^= seed >>> 17; + seed ^= seed << 5; + return Math.abs(seed) % 10; + }; + + for (let n = 0; n < 400; n++) { + const length = 15 + (n % 12); + let literal = String(1 + (nextDigit() % 9)); + for (let d = 1; d < length; d++) literal += String(nextDigit()); + if (n % 3 === 0) literal = `-${literal}`; + + const changes = BigInt(literal) !== emittedInteger(literal); + const verdict = classifyNumberLiteral(literal); + expect(verdict === null, literal).toBe(!changes); + } + }); +}); + +describe('scanLossyRewrites - the eight verified cases', () => { + it('flags large integers that lose digits', () => { + expect(scanLossyRewrites('{"id": 12345678901234567890}')).toEqual({ + kinds: ['unsafe-integer'], + changes: [ + { + kind: 'unsafe-integer', + text: '12345678901234567890', + becomes: '12345678901234567000', + index: 7, + }, + ], + }); + expect(kindsOf('{"big": 9007199254740993}')).toEqual(['unsafe-integer']); + }); + + it('flags an overflow that becomes null', () => { + expect(kindsOf('{"huge": 1e999}')).toEqual(['overflow']); + expect(scanLossyRewrites('{"huge": 1e999}').changes[0].becomes).toBe('null'); + }); + + it('flags minus zero', () => { + expect(kindsOf('{"neg": -0}')).toEqual(['negative-zero']); + }); + + it('flags a duplicate key within one object', () => { + expect(scanLossyRewrites('{"dup": 1, "dup": 2}')).toEqual({ + kinds: ['duplicate-key'], + changes: [{ kind: 'duplicate-key', text: 'dup', index: 11 }], + }); + }); + + it('flags a decimal that loses precision', () => { + expect(kindsOf('{"p": 0.1234567890123456789}')).toEqual(['precision']); + }); + + it('says nothing about 1.0 becoming 1', () => { + expect(scanLossyRewrites('{"x": 1.0}')).toEqual({ kinds: [], changes: [] }); + }); + + it('says nothing about a \\u escape becoming its character', () => { + expect(scanLossyRewrites('{"e": "\\ud83d\\ude00"}')).toEqual({ kinds: [], changes: [] }); + }); +}); + +describe('scanLossyRewrites - false positives', () => { + it('ignores numbers inside strings', () => { + expect(scanLossyRewrites('{"id": "12345678901234567890"}')).toEqual({ + kinds: [], + changes: [], + }); + expect(scanLossyRewrites('"1e999 and -0 and 9007199254740993"')).toEqual({ + kinds: [], + changes: [], + }); + }); + + it('ignores numbers inside smart-quoted strings, which the fixer will re-quote', () => { + expect(kindsOf('{“id”: “12345678901234567890”}')).toEqual([]); + expect(kindsOf("{'id': '1e999'}")).toEqual([]); + }); + + it('ignores numbers inside comments, which the fixer strips', () => { + expect(kindsOf('{"a": 1} // 12345678901234567890')).toEqual([]); + expect(kindsOf('{"a": /* 1e999 */ 1}')).toEqual([]); + }); + + it('does not treat the same key in sibling objects as a duplicate', () => { + expect(kindsOf('[{"a": 1}, {"a": 2}]')).toEqual([]); + expect(kindsOf('{"o": {"a": 1}, "p": {"a": 2}}')).toEqual([]); + }); + + it('does not treat a nested key as a duplicate of its parent', () => { + expect(kindsOf('{"a": {"a": {"a": 1}}}')).toEqual([]); + }); + + it('does not treat an array element matching a key name as a duplicate', () => { + expect(kindsOf('{"a": ["a", "a", "a"]}')).toEqual([]); + }); + + it('leaves an ordinary document alone', () => { + const document = JSON.stringify({ + users: [ + { id: 1, name: 'Ada', score: 99.5, active: true, note: null }, + { id: 2, name: 'Linus', score: -3.25, active: false, note: 'x' }, + ], + totals: { sum: 96.25, count: 2 }, + }); + expect(scanLossyRewrites(document)).toEqual({ kinds: [], changes: [] }); + expect(describe_(document)).toBeNull(); + }); +}); + +describe('scanLossyRewrites - almost-JSON input', () => { + // Format runs the repair pipeline first, so the text scanned here is often + // not valid JSON. The scan has to keep working on exactly those documents. + + it('finds a duplicate key written without quotes', () => { + expect(kindsOf('{a: 1, a: 2,}')).toEqual(['duplicate-key']); + }); + + it('finds a duplicate key across a missing comma', () => { + expect(kindsOf('{"a": 1 "a": 2}')).toEqual(['duplicate-key']); + }); + + it('finds a duplicate key through the fixer’s doubled-brace repair', () => { + // `{{"a":1,"a":2}}` is repaired to `{"a":2}` — the inner object is where both + // keys live either way, so raw-text scoping and parsed scoping agree. + expect(kindsOf('{{"a": 1, "a": 2}}')).toEqual(['duplicate-key']); + }); + + it('finds a big integer in a document with trailing commas and comments', () => { + expect(kindsOf('{\n // note\n "id": 12345678901234567890,\n}')).toEqual(['unsafe-integer']); + }); + + it('makes no claim about a numeric run JSON does not allow', () => { + // The fixer may or may not recover these; either way this module has nothing + // trustworthy to say, and a wrong warning is worse than none. + expect(kindsOf('{"a": +12345678901234567890}')).toEqual([]); + expect(kindsOf('{"a": 012345678901234567890}')).toEqual([]); + expect(kindsOf('{"a": 0x1fffffffffffffff}')).toEqual([]); + }); +}); + +describe('scanLossyRewrites - known misses', () => { + // Pinned rather than hidden: these are the documents where a text scan and + // `JSON.parse` genuinely disagree. + + it('misses duplicates that only exist after the repair joins two documents', () => { + // Two documents side by side. The scan sees two separate objects and reports + // nothing; `JSON.parse` rejects the text outright, so Format never rewrites + // it and the user gets the parse error instead of a silent merge. + expect(kindsOf('{"x": 1} {"x": 2}')).toEqual([]); + }); + + it('misses everything after an unbalanced quote swallows the rest', () => { + // A stray apostrophe in code position opens a string that never closes, so + // the remainder is read as string content. Erring this way is deliberate: + // it costs misses, never false positives. + expect(kindsOf('{"a": don\'t, "b": 1e999}')).toEqual([]); + }); +}); + +describe('scanLossyRewrites - limits', () => { + it('caps the collected changes but never the kinds', () => { + const parts: string[] = []; + for (let i = 0; i < 60; i++) parts.push(`"k${i}": 1234567890123456789${i % 10}`); + parts.push('"z": 1e999'); + const scan = scanLossyRewrites(`{${parts.join(', ')}}`); + + expect(scan.changes).toHaveLength(DEFAULT_LOSSY_CHANGE_LIMIT); + expect(scan.kinds).toEqual(['unsafe-integer', 'overflow']); + }); + + it('honours an explicit limit', () => { + const scan = scanLossyRewrites('[1e999, 1e999, 1e999]', 2); + expect(scan.changes).toHaveLength(2); + expect(scan.kinds).toEqual(['overflow']); + }); + + it('handles an empty document', () => { + expect(scanLossyRewrites('')).toEqual({ kinds: [], changes: [] }); + expect(describe_('')).toBeNull(); + }); + + it('terminates on unbalanced and truncated input', () => { + for (const text of ['{', '[', '}', ']', '{"a":', '{"a": -', '[1,2,', '"unterminated']) { + expect(() => scanLossyRewrites(text), text).not.toThrow(); + } + }); +}); + +describe('describeLossyScan', () => { + it('returns null when Format changes nothing', () => { + expect(describe_('{"a": 1, "b": [2, 3.5]}')).toBeNull(); + }); + + it('names the change and shows one concrete example', () => { + expect(describe_('{"id": 12345678901234567890}')).toBe( + 'large whole numbers lost digits (12345678901234567890 → 12345678901234567000)' + ); + }); + + it('lists every kind in the order it was first seen', () => { + expect(describe_('{"a": -0, "b": 1e999, "b": 2}')).toBe( + 'minus zero became zero, numbers too large for JavaScript became null, ' + + 'duplicate keys were merged, keeping the last value (-0 → 0)' + ); + }); + + it('omits the example when only duplicate keys changed', () => { + expect(describe_('{"dup": 1, "dup": 2}')).toBe( + 'duplicate keys were merged, keeping the last value' + ); + }); + + it('elides an example too long to read', () => { + const long = `1${'0'.repeat(40)}1`; + const rendered = describe_(`{"n": ${long}}`); + expect(rendered).toContain('…'); + expect(rendered).not.toContain(long); + }); + + it('uses the supplied translation for every kind', () => { + const translated = describeLossyScan( + scanLossyRewrites('{"a": -0, "b": 1e999}'), + (key) => `<${key}>` + ); + expect(translated).toBe(', (-0 → 0)'); + }); + + it('has a translation key for every kind it can report', () => { + const kinds: LossyChangeKind[] = [ + 'unsafe-integer', + 'precision', + 'overflow', + 'negative-zero', + 'duplicate-key', + ]; + for (const kind of kinds) { + expect(LOSSY_KIND_TEXT[kind].key.startsWith('format.lossy.'), kind).toBe(true); + expect(LOSSY_KIND_TEXT[kind].fallback.length, kind).toBeGreaterThan(0); + } + }); +}); + +describe('scanLossyRewrites - agrees with an actual round trip', () => { + // The end-to-end property: if the scan says nothing changed, formatting the + // document really does preserve every value. + const faithful = [ + '{"a": 1, "b": -2.5, "c": [0, 1e3, 1.0]}', + '[]', + '{}', + 'null', + '0', + 'false', + '""', + '{"n": 9007199254740991}', + '{"n": -9007199254740991}', + '{"s": "1e999 -0 9007199254740993"}', + ]; + + for (const document of faithful) { + it(`round-trips ${document} unchanged, and the scan agrees`, () => { + const parsed = JSON.parse(document); + // Same value after the round trip, so the scan must stay quiet. + expect(JSON.parse(JSON.stringify(parsed))).toEqual(parsed); + expect(scanLossyRewrites(document).kinds).toEqual([]); + }); + } + + const lossy: [string, LossyChangeKind][] = [ + ['{"a": 12345678901234567890}', 'unsafe-integer'], + ['{"a": 1e999}', 'overflow'], + ['{"a": -0}', 'negative-zero'], + ['{"a": 0.1234567890123456789}', 'precision'], + ['{"a": 1, "a": 2}', 'duplicate-key'], + ]; + + for (const [document, kind] of lossy) { + it(`reports ${kind} for ${document}, which the round trip really does change`, () => { + // Independent confirmation that the document is genuinely rewritten. + expect(JSON.stringify(JSON.parse(document))).not.toBe(document.replace(/\s/g, '')); + expect(scanLossyRewrites(document).kinds).toEqual([kind]); + }); + } + + it('finds nothing in text that JSON.stringify produced', () => { + // The invariant the warning design rests on: `JSON.stringify` output is + // always scan-clean, because everything it could have changed it has + // already changed. So every one of these warns exactly once, however many + // times the user re-formats, re-indents or toggles key sorting afterwards. + const documents = [ + ...faithful, + ...lossy.map(([document]) => document), + '{"b": [1e999, -0, 12345678901234567890, 1e-999]}', + '{"outer": {"a": 1, "a": 2}, "n": 9007199254740993}', + ]; + + for (const document of documents) { + for (const indent of [0, 2, 4, '\t']) { + const formatted = JSON.stringify(JSON.parse(document), null, indent); + expect(scanLossyRewrites(formatted).kinds, formatted).toEqual([]); + } + } + }); +}); diff --git a/src/utils/jsonLossy.ts b/src/utils/jsonLossy.ts new file mode 100644 index 0000000..f95d35a --- /dev/null +++ b/src/utils/jsonLossy.ts @@ -0,0 +1,508 @@ +/** + * What Format is about to change about the user's data. + * + * Format (and Compact) are `JSON.parse` followed by `JSON.stringify`, and that + * round trip is not faithful: every JSON number becomes an IEEE-754 double, and + * every JSON object becomes a JS object, whose keys are unique by definition. + * Measured against this codebase, a valid document is silently rewritten in + * these ways: + * + * {"id": 12345678901234567890} -> 12345678901234567000 digits lost + * {"big": 9007199254740993} -> 9007199254740992 digits lost + * {"p": 0.1234567890123456789} -> 0.12345678901234568 rounded + * {"tiny": 1e-999} -> 0 rounded to zero + * {"huge": 1e999} -> null overflowed + * {"neg": -0} -> 0 sign lost + * {"dup": 1, "dup": 2} -> {"dup": 2} member dropped + * {"x": 1.0} -> 1 re-spelled + * {"e": "\ud83d\ude00"} -> the emoji itself re-spelled + * + * The last two are re-spellings, not losses: `1.0` and `1` denote the same JSON + * number, and an escape sequence and the character it denotes are the same JSON + * string. This module reports the first seven and deliberately says nothing + * about the other two — warning about a change that preserves the value is how + * a warning gets ignored. + * + * A genuinely lossless formatter needs its own number type and its own + * serialiser, which is a much larger piece of work. This is the cheaper and + * more honest half of it: scan the text the user actually typed, *before* it is + * overwritten, and say what the round trip is going to do to it. + * + * Two properties this scan deliberately has: + * + * - It never guesses. A numeric run it cannot read as a JSON number (`+1`, + * `.5`, `01`, `0x1f`, `1.2.3`) produces no claim at all, because a warning + * the user can see is wrong costs more than a miss they never notice. + * - It is tolerant, not a parser. Format runs the repair pipeline first, so the + * text handed here is frequently *not* valid JSON — trailing commas, comments, + * single quotes, unquoted keys. The scanner tracks string boundaries and + * bracket nesting and skips whatever else it meets, so it keeps working on + * the documents Format is most useful for. + * + * One property that falls out of the design and is worth relying on: text that + * `JSON.stringify` produced always scans clean, because everything the round + * trip could change it has already changed. That is what makes it safe to run + * this scan on every action that rewrites the document — the user is warned + * once, and re-formatting, re-indenting or sorting keys afterwards is silent. + * `jsonLossy.test.ts` pins it. + * + * The known limitation of tolerance is duplicate keys: the repair pipeline can + * restructure objects (`{{"a":1,"a":2}}` collapses two brace pairs into one), so + * the object boundaries seen here are the ones the user wrote, not necessarily + * the ones `JSON.parse` sees. `jsonLossy.test.ts` pins the cases that do and do + * not survive that. Number literals are unaffected — a literal is the same + * literal before and after repair. + */ + +/* ------------------------------------------------------------------ * + * Exact decimal values + * ------------------------------------------------------------------ */ + +/** + * A JSON number literal as an exact value: `(negative ? -1 : 1) * digits * 10^exponent`. + * + * Exact is the point. Deciding whether `JSON.parse` changed a literal by + * comparing doubles is circular — both sides are already the same double. The + * only way to see the difference is to compare the literal the user wrote with + * the literal `JSON.stringify` will emit, as mathematical values, with no + * floating point anywhere in the comparison. + */ +export interface ExactDecimal { + negative: boolean; + /** Integer significand with no leading or trailing zeros. Empty means zero. */ + digits: string; + /** Power of ten applied to `digits`. Meaningless when `digits` is empty. */ + exponent: number; +} + +/** The JSON grammar for a number, exactly. */ +const JSON_NUMBER = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/; + +/** + * Read a JSON number literal as an exact value, or `null` if the text is not one. + * + * Returning `null` rather than a best guess is what keeps the scanner quiet + * about `+1`, `.5`, `1.`, `01` and hex: they are not JSON numbers, so this + * module has nothing trustworthy to say about them. + */ +export function exactDecimal(literal: string): ExactDecimal | null { + if (!JSON_NUMBER.test(literal)) return null; + + const negative = literal.charCodeAt(0) === 45; // '-' + const body = negative ? literal.slice(1) : literal; + + const e = body.search(/[eE]/); + const mantissa = e === -1 ? body : body.slice(0, e); + const exponentText = e === -1 ? '0' : body.slice(e + 1); + + const dot = mantissa.indexOf('.'); + const integerPart = dot === -1 ? mantissa : mantissa.slice(0, dot); + const fractionPart = dot === -1 ? '' : mantissa.slice(dot + 1); + + let digits = integerPart + fractionPart; + let exponent = Number(exponentText) - fractionPart.length; + + // Normalise so that equal values have identical representations. + let first = 0; + while (first < digits.length && digits[first] === '0') first++; + digits = digits.slice(first); + + let last = digits.length; + while (last > 0 && digits[last - 1] === '0') { + last--; + exponent++; + } + digits = digits.slice(0, last); + + // Zero has no significand, so its exponent carries no information; pinning it + // makes the representation canonical, so equal values compare equal by shape. + return { negative, digits, exponent: digits === '' ? 0 : exponent }; +} + +/** + * Do two exact decimals denote the same value? + * + * Zero compares by sign as well, so `-0` and `0` are *not* the same value here. + * They are not the same value in JavaScript either (`Object.is(-0, 0)` is false, + * and `1/-0` is `-Infinity`), and losing the sign is one of the rewrites this + * module exists to report. + */ +export function sameExactValue(a: ExactDecimal, b: ExactDecimal): boolean { + const aZero = a.digits === ''; + const bZero = b.digits === ''; + if (aZero || bZero) return aZero && bZero && a.negative === b.negative; + return a.negative === b.negative && a.digits === b.digits && a.exponent === b.exponent; +} + +/* ------------------------------------------------------------------ * + * Classifying one literal + * ------------------------------------------------------------------ */ + +export type LossyChangeKind = + | 'unsafe-integer' + | 'precision' + | 'overflow' + | 'negative-zero' + | 'duplicate-key'; + +export interface NumberVerdict { + kind: Exclude; + /** The literal `JSON.stringify` will emit in place of the original. */ + becomes: string; +} + +/** + * What the round trip does to one number literal — or `null` when it survives + * unchanged, and also `null` when the text is not a JSON number at all. + * + * `String(n)` is used rather than `JSON.stringify(n)` because they agree for + * every finite number, and the non-finite case is handled above it: an overflow + * reaches `JSON.stringify` as `Infinity` and comes back as `null`. + */ +export function classifyNumberLiteral(literal: string): NumberVerdict | null { + const exact = exactDecimal(literal); + if (exact === null) return null; + + const parsed = Number(literal); + if (!Number.isFinite(parsed)) return { kind: 'overflow', becomes: 'null' }; + + const becomes = String(parsed); + const round = exactDecimal(becomes); + // `String` of a finite number is always a JSON number; this only guards the + // engine surprising us. + if (round === null) return null; + if (sameExactValue(exact, round)) return null; + + if (exact.digits === '') return { kind: 'negative-zero', becomes }; + // A non-negative exponent means the value is a whole number, so the digits + // that changed are integer digits past `Number.MAX_SAFE_INTEGER`. + return { kind: exact.exponent >= 0 ? 'unsafe-integer' : 'precision', becomes }; +} + +/* ------------------------------------------------------------------ * + * Scanning a document + * ------------------------------------------------------------------ */ + +export interface LossyChange { + kind: LossyChangeKind; + /** The literal, or the duplicated key name, as it appears in the source. */ + text: string; + /** What the round trip produces. Absent for a duplicate key, which is dropped. */ + becomes?: string; + /** 0-based offset of `text` within the scanned source. */ + index: number; +} + +export interface LossyScan { + /** Every change found, capped at the scan's limit. Empty when nothing changes. */ + changes: LossyChange[]; + /** Distinct kinds present, in first-seen order. Never capped. */ + kinds: LossyChangeKind[]; +} + +/** + * How many individual changes a scan collects. The kinds list is complete + * regardless; this only bounds how much detail is kept for a document where + * thousands of literals change. + */ +export const DEFAULT_LOSSY_CHANGE_LIMIT = 20; + +/** Opening quote -> the characters that can close it. */ +const STRING_DELIMITERS: Record = { + '"': '"', + "'": "'", + '“': '“”', + '”': '“”', + '‘': '‘’', + '’': '‘’', +}; + +const SIMPLE_ESCAPES: Record = { + '"': '"', + "'": "'", + '\\': '\\', + '/': '/', + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', +}; + +function isSpace(ch: string): boolean { + return ch <= ' ' || ch === '\u00a0' || ch === '\ufeff'; +} + +function isDigit(ch: string): boolean { + return ch >= '0' && ch <= '9'; +} + +function isNumberChar(ch: string): boolean { + return isDigit(ch) || ch === '-' || ch === '+' || ch === '.' || ch === 'e' || ch === 'E'; +} + +/** + * Characters that begin a numeric run. `+` and `.` are included even though JSON + * allows neither, so that `+1` and `.5` are read as one run and rejected whole, + * rather than having their leading character skipped and the rest mistaken for a + * well-formed literal. + */ +function startsNumber(ch: string): boolean { + return isDigit(ch) || ch === '-' || ch === '+' || ch === '.'; +} + +function isIdentStart(ch: string): boolean { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_' || ch === '$'; +} + +function isIdentPart(ch: string): boolean { + return isIdentStart(ch) || isDigit(ch); +} + +/** + * Read a string literal and return its *decoded* content. + * + * Decoding matters for key identity: `"ab"` and `"ab"` are the same member + * name after parsing, and comparing the raw spellings would miss that they + * collide. An unterminated string simply runs to the end of the input. + */ +function readString( + source: string, + start: number, + closers: string +): { text: string; next: number } { + let out = ''; + let i = start + 1; + + while (i < source.length) { + const ch = source[i]; + + if (ch === '\\') { + const escaped = source[i + 1]; + if (escaped === undefined) return { text: out, next: source.length }; + if (escaped === 'u' && /^[0-9a-fA-F]{4}$/.test(source.slice(i + 2, i + 6))) { + out += String.fromCharCode(parseInt(source.slice(i + 2, i + 6), 16)); + i += 6; + continue; + } + out += SIMPLE_ESCAPES[escaped] ?? escaped; + i += 2; + continue; + } + + if (closers.includes(ch)) return { text: out, next: i + 1 }; + out += ch; + i++; + } + + return { text: out, next: i }; +} + +interface Frame { + isObject: boolean; + /** Member names already seen in this object. Unused for arrays. */ + keys: Set; + /** True while the next string or bare word would be a member name. */ + awaitingKey: boolean; +} + +/** + * A member's value has just ended, so the next token in an object is a name + * again. Tracking this — rather than waiting for a comma — is what lets the scan + * see the duplicate in `{"a": 1 "a": 2}`, which the repair pipeline turns into a + * genuine object with one member. + */ +function valueEnded(stack: Frame[]): void { + const frame = stack[stack.length - 1]; + if (frame?.isObject === true) frame.awaitingKey = true; +} + +/** + * Find everything `JSON.parse` -> `JSON.stringify` will change about `source`. + * + * Pure and allocation-light: one left-to-right pass, holding only the bracket + * stack and the member names of the objects currently open. + */ +export function scanLossyRewrites( + source: string, + limit: number = DEFAULT_LOSSY_CHANGE_LIMIT +): LossyScan { + const changes: LossyChange[] = []; + const kinds: LossyChangeKind[] = []; + + const record = (change: LossyChange): void => { + if (!kinds.includes(change.kind)) kinds.push(change.kind); + if (changes.length < limit) changes.push(change); + }; + + const stack: Frame[] = []; + const len = source.length; + let i = 0; + + /** Handle a string or bare word. Returns true when it was a member name. */ + const noteWord = (name: string, index: number): boolean => { + const frame = stack[stack.length - 1]; + if (frame === undefined || !frame.isObject || !frame.awaitingKey) return false; + frame.awaitingKey = false; + if (frame.keys.has(name)) { + record({ kind: 'duplicate-key', text: name, index }); + return true; + } + frame.keys.add(name); + return true; + }; + + while (i < len) { + const ch = source[i]; + + if (isSpace(ch)) { + i++; + continue; + } + + // Comments: the repair pipeline strips them, so a number inside one is not + // part of the document and must not be reported. + if (ch === '/' && source[i + 1] === '/') { + i += 2; + while (i < len && source[i] !== '\n') i++; + continue; + } + if (ch === '/' && source[i + 1] === '*') { + i += 2; + while (i < len && !(source[i] === '*' && source[i + 1] === '/')) i++; + i = Math.min(i + 2, len); + continue; + } + + const closers = STRING_DELIMITERS[ch]; + if (closers !== undefined) { + const start = i; + const read = readString(source, i, closers); + i = read.next; + if (!noteWord(read.text, start)) valueEnded(stack); + continue; + } + + if (ch === '{' || ch === '[') { + stack.push({ isObject: ch === '{', keys: new Set(), awaitingKey: ch === '{' }); + i++; + continue; + } + + if (ch === '}' || ch === ']') { + stack.pop(); + // The container that just closed was itself a member value. + valueEnded(stack); + i++; + continue; + } + + if (ch === ',') { + const frame = stack[stack.length - 1]; + if (frame !== undefined) frame.awaitingKey = frame.isObject; + i++; + continue; + } + + if (ch === ':') { + const frame = stack[stack.length - 1]; + if (frame !== undefined) frame.awaitingKey = false; + i++; + continue; + } + + if (startsNumber(ch)) { + const start = i; + while (i < len && isNumberChar(source[i])) i++; + const literal = source.slice(start, i); + const frame = stack[stack.length - 1]; + // A number can never be a member name, so a run in key position is + // malformed and gets no claim. Neither does a run that runs straight into + // a word (`12abc`, `0x1f`): that is a bare token the repair pipeline will + // quote, not a number. + const inKeyPosition = frame?.isObject === true && frame.awaitingKey; + const runsIntoWord = i < len && isIdentStart(source[i]); + if (!inKeyPosition && !runsIntoWord) { + const verdict = classifyNumberLiteral(literal); + if (verdict !== null) { + record({ kind: verdict.kind, text: literal, becomes: verdict.becomes, index: start }); + } + } + if (!inKeyPosition) valueEnded(stack); + continue; + } + + if (isIdentStart(ch)) { + const start = i; + while (i < len && isIdentPart(source[i])) i++; + if (!noteWord(source.slice(start, i), start)) valueEnded(stack); + continue; + } + + i++; + } + + return { changes, kinds }; +} + +/* ------------------------------------------------------------------ * + * Describing a scan to the user + * ------------------------------------------------------------------ */ + +/** Translation key and English default for each kind of change. */ +export const LOSSY_KIND_TEXT: Record = { + 'unsafe-integer': { + key: 'format.lossy.unsafeInteger', + fallback: 'large whole numbers lost digits', + }, + precision: { + key: 'format.lossy.precision', + fallback: 'decimal numbers were rounded', + }, + overflow: { + key: 'format.lossy.overflow', + fallback: 'numbers too large for JavaScript became null', + }, + 'negative-zero': { + key: 'format.lossy.negativeZero', + fallback: 'minus zero became zero', + }, + 'duplicate-key': { + key: 'format.lossy.duplicateKey', + fallback: 'duplicate keys were merged, keeping the last value', + }, +}; + +/** Longest literal shown in the example; longer ones are elided in the middle. */ +const EXAMPLE_MAX_LENGTH = 24; + +function elide(text: string): string { + if (text.length <= EXAMPLE_MAX_LENGTH) return text; + return `${text.slice(0, EXAMPLE_MAX_LENGTH - 1)}…`; +} + +/** + * Turn a scan into one sentence fragment for the toast, or `null` when the + * document survives Format untouched. + * + * `translate` is passed in rather than imported so this stays pure and testable: + * the component supplies i18next, the tests supply the English defaults. + */ +export function describeLossyScan( + scan: LossyScan, + translate: (key: string, fallback: string) => string +): string | null { + if (scan.kinds.length === 0) return null; + + const phrases = scan.kinds.map((kind) => { + const text = LOSSY_KIND_TEXT[kind]; + return translate(text.key, text.fallback); + }); + + // One concrete before/after does more than any amount of prose. Duplicate keys + // have no "after" to show, so the example comes from the first number change. + const example = scan.changes.find((change) => change.becomes !== undefined); + const detail = phrases.join(', '); + if (example === undefined) return detail; + return `${detail} (${elide(example.text)} → ${elide(example.becomes as string)})`; +} diff --git a/src/utils/jsonPath.test.ts b/src/utils/jsonPath.test.ts new file mode 100644 index 0000000..dd0a2d3 --- /dev/null +++ b/src/utils/jsonPath.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from 'vitest'; +import { + collectExpandablePaths, + defaultExpandedPaths, + entriesOf, + formatPath, + formatSegment, + getAtPath, + isPathPrefix, + type PathSegment, + parsePath, + pathsEqual, + remapExpandedAfterRename, + renameKeyInObject, + setAtPath, +} from './jsonPath'; +import { MAX_JSON_DEPTH, nestDeeply, runDepthGuarded } from './jsonWalk'; + +describe('formatPath / parsePath', () => { + const roundTrip = (segments: PathSegment[]) => { + const formatted = formatPath(segments); + expect(parsePath(formatted)).toEqual(segments); + return formatted; + }; + + it('renders identifier keys in dot form and everything else in bracket form', () => { + expect(formatPath(['a', 'b'])).toBe('root.a.b'); + expect(formatPath(['$ref', '_id'])).toBe('root.$ref._id'); + expect(formatPath(['a.b'])).toBe('root["a.b"]'); + expect(formatPath([0, 'a', 12])).toBe('root[0].a[12]'); + expect(formatPath([])).toBe('root'); + }); + + it('round-trips keys containing path punctuation', () => { + roundTrip(['a.b']); + roundTrip(['a[0]']); + roundTrip(['a]b[c']); + roundTrip(['root.a']); + roundTrip(['.']); + roundTrip(['[', ']', '.']); + roundTrip(['a', 'b.c', 0, 'd[1]']); + }); + + it('round-trips keys containing quotes, escapes, whitespace and unicode', () => { + roundTrip(['he said "hi"']); + roundTrip(['back\\slash']); + roundTrip(['line\nbreak\ttab']); + roundTrip(['']); + roundTrip([' ']); + roundTrip(['clé-unicode-日本語-😀']); + roundTrip(['__proto__', 'constructor', 'prototype']); + }); + + it('keeps array indices and numeric-looking object keys apart', () => { + expect(formatSegment(0)).toBe('[0]'); + expect(formatSegment('0')).toBe('["0"]'); + expect(formatPath([0])).not.toBe(formatPath(['0'])); + expect(parsePath('root[0]')).toEqual([0]); + expect(parsePath('root["0"]')).toEqual(['0']); + }); + + it('does not collide {"a.b": 1} with {"a": {"b": 1}} — the BUG 3 case', () => { + const dotted = formatPath(['a.b']); + const nested = formatPath(['a', 'b']); + + expect(dotted).not.toBe(nested); + expect(parsePath(dotted)).toEqual(['a.b']); + expect(parsePath(nested)).toEqual(['a', 'b']); + }); + + it('returns null for malformed path strings instead of guessing', () => { + expect(parsePath('root.')).toBeNull(); + expect(parsePath('root[')).toBeNull(); + expect(parsePath('root[]')).toBeNull(); + expect(parsePath('root[01]')).toBeNull(); + expect(parsePath('root[-1]')).toBeNull(); + expect(parsePath('root[1.5]')).toBeNull(); + expect(parsePath('root["unterminated]')).toBeNull(); + expect(parsePath('root["a"')).toBeNull(); + expect(parsePath('root.a-b')).toBeNull(); + expect(parsePath('elsewhere.a')).toBeNull(); + expect(parsePath('rootless.a')).toBeNull(); + }); +}); + +describe('path comparisons', () => { + it('compares paths segment-wise, not as strings', () => { + expect(pathsEqual(['a', 'b'], ['a', 'b'])).toBe(true); + expect(pathsEqual(['a.b'], ['a', 'b'])).toBe(false); + expect(pathsEqual([0], ['0'])).toBe(false); + }); + + it('treats a path as a prefix of itself and of its descendants only', () => { + expect(isPathPrefix(['a'], ['a', 'b'])).toBe(true); + expect(isPathPrefix(['a'], ['a'])).toBe(true); + expect(isPathPrefix(['a'], ['ab'])).toBe(false); + expect(isPathPrefix(['a', 'b'], ['a'])).toBe(false); + }); +}); + +describe('entriesOf', () => { + it('yields numeric segments for arrays and string segments for objects', () => { + expect(entriesOf(['x', 'y'])).toEqual([ + [0, 'x'], + [1, 'y'], + ]); + expect(entriesOf({ a: 1 })).toEqual([['a', 1]]); + expect(entriesOf(42)).toEqual([]); + expect(entriesOf(null)).toEqual([]); + }); +}); + +describe('getAtPath', () => { + it('reads nested values through both object and array segments', () => { + const doc = { a: { b: [10, { c: 'x' }] } }; + expect(getAtPath(doc, ['a', 'b', 1, 'c'])).toBe('x'); + expect(getAtPath(doc, [])).toBe(doc); + }); + + it('distinguishes a dotted key from a nested object', () => { + const doc = { 'a.b': 1, a: { b: 2 } }; + expect(getAtPath(doc, ['a.b'])).toBe(1); + expect(getAtPath(doc, ['a', 'b'])).toBe(2); + }); + + it('never reaches through the prototype chain', () => { + expect(getAtPath({}, ['toString'])).toBeUndefined(); + expect(getAtPath({}, ['constructor', 'name'])).toBeUndefined(); + expect(getAtPath([1, 2], [5])).toBeUndefined(); + expect(getAtPath([1, 2], ['length'])).toBeUndefined(); + }); +}); + +describe('setAtPath', () => { + it('writes object and array slots in place', () => { + const doc = { a: { b: [1, 2] } }; + expect(setAtPath(doc, ['a', 'b', 0], 9)).toBe(true); + expect(doc.a.b[0]).toBe(9); + }); + + it('refuses paths that do not address an existing slot', () => { + const doc = { a: 1 }; + expect(setAtPath(doc, [], 2)).toBe(false); + expect(setAtPath(doc, ['missing'], 2)).toBe(false); + expect(setAtPath(doc, ['a', 'b'], 2)).toBe(false); + expect(setAtPath([1], [3], 2)).toBe(false); + }); + + it('edits a dotted key without touching the same-looking nested value', () => { + const doc = { 'a.b': 1, a: { b: 2 } }; + setAtPath(doc, ['a.b'], 99); + expect(doc).toEqual({ 'a.b': 99, a: { b: 2 } }); + }); + + it('writes a __proto__ key as an own property instead of polluting the prototype', () => { + const doc = JSON.parse('{"__proto__": 1}'); + expect(setAtPath(doc, ['__proto__'], { polluted: true })).toBe(true); + expect(JSON.stringify(doc)).toBe('{"__proto__":{"polluted":true}}'); + expect(Object.getPrototypeOf(doc)).toBe(Object.prototype); + expect(({} as Record).polluted).toBeUndefined(); + }); +}); + +describe('renameKeyInObject', () => { + it('preserves key order', () => { + const renamed = renameKeyInObject({ a: 1, b: 2, c: 3 }, 'b', 'z'); + expect(Object.keys(renamed)).toEqual(['a', 'z', 'c']); + expect(renamed.z).toBe(2); + }); + + it('round-trips a __proto__ key through clone → rename → stringify', () => { + const source = JSON.parse('{"a": 1, "b": 2}'); + const clone = structuredClone(source); + const renamed = renameKeyInObject(clone, 'b', '__proto__'); + + expect(JSON.stringify(renamed)).toBe('{"a":1,"__proto__":2}'); + expect(Object.getPrototypeOf(renamed)).toBe(Object.prototype); + expect(Object.hasOwn(renamed, '__proto__')).toBe(true); + }); + + it('renames away from a __proto__ key too', () => { + const clone = structuredClone(JSON.parse('{"__proto__": {"a": 1}, "b": 2}')); + const renamed = renameKeyInObject(clone, '__proto__', 'safe'); + expect(JSON.stringify(renamed)).toBe('{"safe":{"a":1},"b":2}'); + }); +}); + +describe('defaultExpandedPaths', () => { + it('uses array syntax for a root-level array so the rendered rows match', () => { + const paths = defaultExpandedPaths([{ a: 1 }, [1, 2], 'leaf']); + expect(paths.has('root[0]')).toBe(true); + expect(paths.has('root[1]')).toBe(true); + // Leaves are not expandable, so they are never pre-expanded. + expect(paths.has('root[2]')).toBe(false); + expect(paths.has('root.0')).toBe(false); + }); + + it('expands the first ten top-level containers only', () => { + const doc = Object.fromEntries( + Array.from({ length: 12 }, (_, index) => [`k${index}`, { nested: true }]) + ); + const paths = defaultExpandedPaths(doc); + expect(paths.has('root.k9')).toBe(true); + expect(paths.has('root.k10')).toBe(false); + }); + + it('escapes second-level keys that contain path punctuation', () => { + const paths = defaultExpandedPaths({ a: { 'b.c': { deep: 1 } } }); + expect(paths.has('root.a')).toBe(true); + expect(paths.has('root.a["b.c"]')).toBe(true); + expect(paths.has('root.a.b.c')).toBe(false); + }); +}); + +describe('collectExpandablePaths', () => { + it('collects every container path and nothing else', () => { + const paths = collectExpandablePaths({ a: { b: [{ c: 1 }] }, d: 5 }); + expect([...paths].sort()).toEqual(['root.a', 'root.a.b', 'root.a.b[0]']); + }); + + it('collects containers inside a root-level array', () => { + const paths = collectExpandablePaths([{ a: 1 }, 2]); + expect([...paths]).toEqual(['root[0]']); + }); +}); + +describe('remapExpandedAfterRename', () => { + it('rewrites the renamed node and its descendants, leaving siblings alone', () => { + const paths = new Set(['root.a', 'root.a.b', 'root.a.b[0]', 'root.ab', 'root.z']); + const remapped = remapExpandedAfterRename(paths, [], 'a', 'renamed'); + + expect([...remapped].sort()).toEqual([ + 'root.ab', + 'root.renamed', + 'root.renamed.b', + 'root.renamed.b[0]', + 'root.z', + ]); + }); + + it('only rewrites below the parent that owns the renamed key', () => { + const paths = new Set(['root.x.a', 'root.y.a']); + const remapped = remapExpandedAfterRename(paths, ['x'], 'a', 'b'); + expect([...remapped].sort()).toEqual(['root.x.b', 'root.y.a']); + }); + + it('escapes a new key that contains path punctuation', () => { + const remapped = remapExpandedAfterRename(new Set(['root.a', 'root.a.b']), [], 'a', 'a.b'); + expect([...remapped].sort()).toEqual(['root["a.b"]', 'root["a.b"].b']); + }); + + it('passes through entries it cannot parse', () => { + const remapped = remapExpandedAfterRename(new Set(['not-a-path']), [], 'a', 'b'); + expect([...remapped]).toEqual(['not-a-path']); + }); +}); + +describe('collectExpandablePaths - depth guard', () => { + it('reports a 100,000-level document instead of throwing RangeError', () => { + expect(runDepthGuarded(() => collectExpandablePaths(nestDeeply(100_000, 'object')))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + }); + + it('collects every container in an ordinary document', () => { + expect(collectExpandablePaths({ a: { b: [1, { c: 2 }] } })).toEqual( + new Set(['root.a', 'root.a.b', 'root.a.b[1]']) + ); + }); + + it('leaves wide-but-shallow documents alone', () => { + const wide = Array.from({ length: 1_000 }, () => ({})); + expect(collectExpandablePaths(wide).size).toBe(1_000); + }); +}); diff --git a/src/utils/jsonPath.ts b/src/utils/jsonPath.ts new file mode 100644 index 0000000..74f03f4 --- /dev/null +++ b/src/utils/jsonPath.ts @@ -0,0 +1,276 @@ +/** + * Structured JSON paths. + * + * A path is an array of segments: a `string` segment addresses an object key and + * a `number` segment addresses an array index. Keeping paths structured is what + * makes `{"a.b": 1}` and `{"a": {"b": 1}}` distinguishable — the legacy + * `root.a.b` string form collapsed them onto the same key, so expanding or + * editing one silently hit the other. + * + * `formatPath` renders a segment array as a display string that is *injective*: + * keys that are not plain identifiers are emitted in bracket-quoted JSON form, + * so the display string can be used as a React key / Set member and parsed back + * with `parsePath` without loss. + * + * These helpers are intentionally free of React so other views that still tear + * paths apart with `split(/\.|\[|\]/)` (SearchView, at time of writing) can adopt + * them without any further refactor. + */ + +import { assertJsonDepth } from './jsonWalk'; + +export type PathSegment = string | number; +export type JsonPath = readonly PathSegment[]; + +const ROOT_LABEL = 'root'; +const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +const INDEX = /^(0|[1-9][0-9]*)$/; + +/** Number of top-level entries expanded by default. */ +const DEFAULT_TOP_LEVEL = 10; +/** Number of top-level entries whose children are also expanded by default. */ +const DEFAULT_SECOND_LEVEL_PARENTS = 3; +/** Number of children expanded per second-level parent. */ +const DEFAULT_SECOND_LEVEL = 3; + +export function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isJsonContainer(value: unknown): value is Record | unknown[] { + return typeof value === 'object' && value !== null; +} + +/** + * Child entries of a container as `[segment, value]` pairs — numeric segments for + * arrays, string segments for objects. Non-containers have no entries. + */ +export function entriesOf(value: unknown): [PathSegment, unknown][] { + if (Array.isArray(value)) return value.map((item, index) => [index, item]); + if (isJsonObject(value)) return Object.entries(value); + return []; +} + +export function formatSegment(segment: PathSegment): string { + if (typeof segment === 'number') return `[${segment}]`; + if (IDENTIFIER.test(segment)) return `.${segment}`; + return `[${JSON.stringify(segment)}]`; +} + +/** Display / identity string for a path. Injective: distinct paths never collide. */ +export function formatPath(segments: JsonPath, root: string = ROOT_LABEL): string { + let out = root; + for (const segment of segments) out += formatSegment(segment); + return out; +} + +/** Inverse of {@link formatPath}. Returns `null` for anything malformed. */ +export function parsePath(path: string, root: string = ROOT_LABEL): PathSegment[] | null { + if (!path.startsWith(root)) return null; + const segments: PathSegment[] = []; + let i = root.length; + + while (i < path.length) { + const char = path[i]; + + if (char === '.') { + i += 1; + const start = i; + while (i < path.length && path[i] !== '.' && path[i] !== '[') i += 1; + const name = path.slice(start, i); + if (!IDENTIFIER.test(name)) return null; + segments.push(name); + continue; + } + + if (char !== '[') return null; + i += 1; + + if (path[i] === '"') { + let j = i + 1; + let closed = false; + while (j < path.length) { + if (path[j] === '\\') { + j += 2; + continue; + } + if (path[j] === '"') { + closed = true; + break; + } + j += 1; + } + if (!closed || path[j + 1] !== ']') return null; + let key: unknown; + try { + key = JSON.parse(path.slice(i, j + 1)); + } catch { + return null; + } + if (typeof key !== 'string') return null; + segments.push(key); + i = j + 2; + continue; + } + + const end = path.indexOf(']', i); + if (end === -1) return null; + const digits = path.slice(i, end); + if (!INDEX.test(digits)) return null; + segments.push(Number(digits)); + i = end + 1; + } + + return segments; +} + +export function pathsEqual(a: JsonPath, b: JsonPath): boolean { + if (a.length !== b.length) return false; + return a.every((segment, index) => segment === b[index]); +} + +/** True when `prefix` addresses `path` itself or one of its ancestors. */ +export function isPathPrefix(prefix: JsonPath, path: JsonPath): boolean { + if (prefix.length > path.length) return false; + return prefix.every((segment, index) => segment === path[index]); +} + +/** + * Assign an own, enumerable, writable property. Plain `obj[key] = value` runs the + * inherited `__proto__` setter, so a key named `__proto__` would mutate the + * prototype instead of round-tripping through `JSON.stringify`. + */ +function defineOwn(target: Record, key: string, value: unknown): void { + Object.defineProperty(target, key, { + value, + writable: true, + enumerable: true, + configurable: true, + }); +} + +/** Read a value by path. Own properties only — never walks the prototype chain. */ +export function getAtPath(root: unknown, segments: JsonPath): unknown { + let current: unknown = root; + for (const segment of segments) { + if (typeof segment === 'number') { + if (!Array.isArray(current) || segment < 0 || segment >= current.length) return undefined; + current = current[segment]; + continue; + } + if (!isJsonObject(current) || !Object.hasOwn(current, segment)) return undefined; + current = current[segment]; + } + return current; +} + +/** + * Write a value by path, mutating `root` in place. Returns `false` when the path + * does not address an existing slot (empty path included — there is no container + * to write into, so the caller must replace the root itself). + */ +export function setAtPath(root: unknown, segments: JsonPath, value: unknown): boolean { + if (segments.length === 0) return false; + const parent = getAtPath(root, segments.slice(0, -1)); + const last = segments[segments.length - 1]; + + if (typeof last === 'number') { + if (!Array.isArray(parent) || last < 0 || last >= parent.length) return false; + parent[last] = value; + return true; + } + + if (!isJsonObject(parent) || !Object.hasOwn(parent, last)) return false; + defineOwn(parent, last, value); + return true; +} + +/** + * Rebuild `source` with `oldKey` renamed to `newKey`, preserving key order. + * `Object.fromEntries` uses CreateDataProperty, so `__proto__` survives as an own + * property rather than being swallowed by the prototype setter. + */ +export function renameKeyInObject( + source: Record, + oldKey: string, + newKey: string +): Record { + return Object.fromEntries( + Object.entries(source).map(([key, value]) => [key === oldKey ? newKey : key, value]) + ); +} + +/** + * Paths of every container node, used by "expand all". + * + * Throws `JsonDepthLimitError` past `MAX_JSON_DEPTH` rather than recursing into a + * document deep enough to blow the stack. TreeView refuses to render such a + * document at all, so in practice this guard is only ever a backstop. + */ +export function collectExpandablePaths(json: unknown): Set { + const paths = new Set(); + const walk = (value: unknown, segments: PathSegment[], depth: number) => { + assertJsonDepth(depth); + for (const [segment, child] of entriesOf(value)) { + const next = [...segments, segment]; + if (isJsonContainer(child)) { + paths.add(formatPath(next)); + walk(child, next, depth + 1); + } + } + }; + walk(json, [], 0); + return paths; +} + +/** + * Paths expanded when a document is first shown: the first ten top-level + * containers, plus the first three children of the first three of those. + */ +export function defaultExpandedPaths(json: unknown): Set { + const paths = new Set(); + const topLevel = entriesOf(json); + + topLevel.slice(0, DEFAULT_TOP_LEVEL).forEach(([segment, value]) => { + if (isJsonContainer(value)) paths.add(formatPath([segment])); + }); + + topLevel.slice(0, DEFAULT_SECOND_LEVEL_PARENTS).forEach(([segment, value]) => { + if (!isJsonContainer(value)) return; + entriesOf(value) + .slice(0, DEFAULT_SECOND_LEVEL) + .forEach(([childSegment, childValue]) => { + if (isJsonContainer(childValue)) paths.add(formatPath([segment, childSegment])); + }); + }); + + return paths; +} + +/** + * Rewrite expanded-node keys after `parentPath[oldKey]` is renamed to `newKey`, + * covering the renamed node itself and every descendant. Unparseable entries are + * passed through untouched. + */ +export function remapExpandedAfterRename( + paths: Iterable, + parentPath: JsonPath, + oldKey: string, + newKey: string +): Set { + const renamedPath = [...parentPath, oldKey]; + const remapped = new Set(); + + for (const pathKey of paths) { + const segments = parsePath(pathKey); + if (!segments || !isPathPrefix(renamedPath, segments)) { + remapped.add(pathKey); + continue; + } + const next = [...segments]; + next[parentPath.length] = newKey; + remapped.add(formatPath(next)); + } + + return remapped; +} diff --git a/src/utils/jsonSearch.test.ts b/src/utils/jsonSearch.test.ts new file mode 100644 index 0000000..55f97c8 --- /dev/null +++ b/src/utils/jsonSearch.test.ts @@ -0,0 +1,635 @@ +import { describe, expect, it } from 'vitest'; +import { + analyzeJsonStructure, + buildSuggestions, + compileMatcher, + compileUserRegex, + DEFAULT_TYPE_FILTERS, + highlightSegments, + MAX_PATTERN_LENGTH, + probeRegexCost, + REGEX_INPUT_LIMIT, + REGEX_PROBE_BUDGET_MS, + searchJson, + tokenizeWords, +} from './jsonSearch'; +import { MAX_JSON_DEPTH, nestDeeply } from './jsonWalk'; + +const sample = { + user: { + name: 'Ada Lovelace', + nickname: 'ada', + address: { city: 'London', name: 'home' }, + }, + admin: { name: 'Grace' }, +}; + +describe('searchJson - path pattern filter', () => { + it('matches the candidate path, not its container (the user\\..* case)', () => { + const outcome = searchJson(sample, { query: 'name', pathPattern: 'user\\..*' }); + const paths = outcome.results.map((r) => r.path); + // Before the fix the regex was tested against the container's path, which + // is '' at the root, so this returned zero results. + expect(paths).toContain('user.name'); + expect(paths).toContain('user.nickname'); + expect(paths).not.toContain('admin.name'); + }); + + it('supports anchored patterns that can only match a leaf', () => { + const outcome = searchJson(sample, { query: 'name', pathPattern: 'user\\.name$' }); + expect(outcome.results.map((r) => r.path)).toEqual(['user.name']); + }); + + it('finds deeply nested candidates even though no ancestor path matches', () => { + const nested = { a: { b: { user: { name: 'Ada' } } } }; + const outcome = searchJson(nested, { query: 'name', pathPattern: 'user\\.name$' }); + // Pruning the walk on the path pattern would resurrect the original bug. + expect(outcome.results.map((r) => r.path)).toEqual(['a.b.user.name']); + }); + + it('reports an invalid path pattern instead of silently ignoring it', () => { + const outcome = searchJson(sample, { query: 'name', pathPattern: '([' }); + expect(outcome.pathPatternError).toBeTruthy(); + }); +}); + +describe('searchJson - exclude pattern', () => { + it('drops excluded candidates', () => { + const outcome = searchJson(sample, { query: 'name', excludePattern: '^admin' }); + expect(outcome.results.map((r) => r.path)).not.toContain('admin.name'); + expect(outcome.results.map((r) => r.path)).toContain('user.name'); + }); + + it('prunes the excluded subtree rather than filtering at the end', () => { + const doc = { keep: { name: 'a' }, skip: { deep: { name: 'b', other: 'c' } } }; + const withExclude = searchJson(doc, { query: 'name', excludePattern: '^skip' }); + const withoutExclude = searchJson(doc, { query: 'name' }); + expect(withExclude.results.map((r) => r.path)).toEqual(['keep.name']); + // The excluded branch costs nothing: fewer nodes are visited. + expect(withExclude.visited).toBeLessThan(withoutExclude.visited); + }); + + it('prunes excluded array elements', () => { + const doc = { items: [{ name: 'a' }, { name: 'b' }] }; + const outcome = searchJson(doc, { query: 'name', excludePattern: '\\[1\\]' }); + expect(outcome.results.map((r) => r.path)).toEqual(['items[0].name']); + }); +}); + +describe('searchJson - match modes', () => { + const doc = { Title: 'Hello World', other: 'title case' }; + + it('contains is case-insensitive by default and case-sensitive on request', () => { + expect(searchJson(doc, { query: 'title' }).results.length).toBe(2); + expect( + searchJson(doc, { query: 'title', caseSensitive: true }).results.map((r) => r.path) + ).toEqual(['other']); + }); + + it('exact matches whole strings only', () => { + expect(searchJson(doc, { query: 'title', mode: 'exact' }).results.map((r) => r.path)).toEqual([ + 'Title', + ]); + expect( + searchJson(doc, { query: 'title', mode: 'exact', caseSensitive: true }).results + ).toHaveLength(0); + }); + + it('starts and ends anchor to the right edge', () => { + expect(searchJson(doc, { query: 'hello', mode: 'starts' }).results.map((r) => r.path)).toEqual([ + 'Title', + ]); + expect(searchJson(doc, { query: 'world', mode: 'ends' }).results.map((r) => r.path)).toEqual([ + 'Title', + ]); + expect( + searchJson(doc, { query: 'World', mode: 'ends', caseSensitive: true }).results.map( + (r) => r.path + ) + ).toEqual(['Title']); + expect(searchJson(doc, { query: 'world', mode: 'ends', caseSensitive: true }).results).toEqual( + [] + ); + }); + + it('regex mode honours case sensitivity', () => { + expect(searchJson(doc, { query: '^hello', mode: 'regex' }).results.map((r) => r.path)).toEqual([ + 'Title', + ]); + expect( + searchJson(doc, { query: '^hello', mode: 'regex', caseSensitive: true }).results + ).toEqual([]); + }); + + it('reports an invalid query regex through queryError', () => { + const outcome = searchJson(doc, { query: '([', mode: 'regex' }); + expect(outcome.queryError).toBeTruthy(); + expect(outcome.results).toEqual([]); + }); + + it('treats a scalar root as a document, not as nothing loaded', () => { + // `null`, `0`, `false` and `""` are all valid JSON documents. There is + // nothing to find inside one, but a bad pattern still has to be reported + // rather than swallowed by an early return. + for (const document of [null, 0, false, '']) { + expect(searchJson(document, { query: 'x' }).results).toEqual([]); + expect(searchJson(document, { query: '([', mode: 'regex' }).queryError?.code).toBe('invalid'); + } + }); +}); + +describe('compileMatcher - statefulness', () => { + it('returns the same answer when a compiled regex matcher is reused', () => { + const { matcher } = compileMatcher('a', 'regex', false); + // A `g` flag would carry lastIndex across calls and alternate true/false. + expect(matcher?.matches('abc')).toBe(true); + expect(matcher?.matches('abc')).toBe(true); + expect(matcher?.matches('abc')).toBe(true); + }); + + it('matches every sibling key when the walker reuses one compiled regex', () => { + const doc = { aa: 1, ab: 2, ac: 3, ad: 4 }; + const outcome = searchJson(doc, { query: '^a', mode: 'regex' }); + expect(outcome.results).toHaveLength(4); + }); +}); + +describe('compileUserRegex - cost guards', () => { + it('rejects nested quantifiers that can backtrack catastrophically', () => { + expect(compileUserRegex('(a+)+$', '').regex).toBeNull(); + expect(compileUserRegex('([a-z]+\\d)*', '').error?.code).toBe('nested-quantifier'); + }); + + it('still compiles ordinary patterns', () => { + expect(compileUserRegex('(foo|bar)+', '').regex).toBeInstanceOf(RegExp); + expect(compileUserRegex('^user\\.\\d+$', '').regex).toBeInstanceOf(RegExp); + expect(compileUserRegex('\\._id$', 'i').regex).toBeInstanceOf(RegExp); + expect(compileUserRegex('[a-z]+@[a-z]+\\.[a-z]{2,}', '').regex).toBeInstanceOf(RegExp); + expect(compileUserRegex('(\\d{1,3}\\.){3}\\d{1,3}', '').regex).toBeInstanceOf(RegExp); + expect(compileUserRegex('(users|orders)\\[\\d+\\]\\.(id|name)$', '').regex).toBeInstanceOf( + RegExp + ); + }); + + it('rejects absurdly long patterns', () => { + const error = compileUserRegex('a'.repeat(500), '').error; + expect(error?.code).toBe('too-long'); + expect(error?.length).toBe(500); + expect(error?.limit).toBe(MAX_PATTERN_LENGTH); + }); + + it('reports an uncompilable pattern with the engine own message attached', () => { + const error = compileUserRegex('([', '').error; + expect(error?.code).toBe('invalid'); + expect(error?.detail).toBeTruthy(); + }); + + it('measures the families the syntactic check cannot see', () => { + // Each of these is catastrophic and each slips past NESTED_QUANTIFIER: the + // character class in that pattern cannot span nested parentheses, cross an + // alternation, or look inside a brace quantifier. The cost probe runs them + // instead of reading them, so it does not have to recognise the shape. + for (const pattern of ['((a+))+$', '(a|a)*$', '(a{2,})+$', '^(a|a?)+$']) { + expect(compileUserRegex(pattern, '').error?.code).toBe('too-slow'); + } + }); + + it('leaves the shapes the syntactic check does catch to it', () => { + // The cheap check runs first precisely so the probe never has to measure + // the deeply nested shapes, which are the most expensive to measure. + for (const pattern of ['(a+)+$', '(?:a+)+$', '(x+x+)+y', '(((a+)+)+)+$']) { + expect(compileUserRegex(pattern, '').error?.code).toBe('nested-quantifier'); + } + }); + + it('gives up between two probe calls, not after the whole ladder', () => { + // A clock that jumps past the budget on its second reading. The ladder has + // fifteen lengths and two strings each; reading the clock only twice proves + // it stopped after the very first call, which is what keeps the probe from + // becoming the freeze it exists to prevent. + let reads = 0; + const now = () => (reads++ === 0 ? 0 : 10_000); + expect(probeRegexCost(/^a+$/, '^a+$', now)).toBe(true); + expect(reads).toBe(2); + }); + + it('costs a fraction of its own budget on ordinary patterns', () => { + // The probe is only tolerable if it is invisible on real queries: this is + // the margin that keeps a loaded machine from refusing them. + const startedAt = Date.now(); + for (let i = 0; i < 20; i++) compileUserRegex('^(GET|POST)\\s/api/v\\d+/users$', 'i'); + expect(Date.now() - startedAt).toBeLessThan(20 * REGEX_PROBE_BUDGET_MS); + }); +}); + +describe('regex mode - hostile patterns', () => { + // 4,000 a's followed by one character that cannot match: the shape every + // catastrophic-backtracking demo uses, and long enough that a single + // unguarded evaluation would not finish this decade. + const hostileValue = `${'a'.repeat(4_000)}!`; + const doc = { note: hostileValue, other: 'plain' }; + + it.each(['((a+))+$', '(a|a)*$'])('refuses %s instead of hanging on it', (pattern) => { + const startedAt = Date.now(); + const outcome = searchJson(doc, { query: pattern, mode: 'regex' }); + const elapsed = Date.now() - startedAt; + + // Assert the mechanism as well as the clock: if some later change makes + // these patterns cheap for an unrelated reason, the timing alone would keep + // passing while the guard was gone. + expect(outcome.queryError?.code).toBe('too-slow'); + expect(outcome.results).toEqual([]); + // Unguarded this is hours; the bound is generous so a busy CI box is fine. + expect(elapsed).toBeLessThan(1_000); + }); + + it('runs an accepted pattern against only the first few KB of a value', () => { + const long = { note: `${'a'.repeat(REGEX_INPUT_LIMIT)}needle` }; + // The clamp is the backstop for whatever the heuristics let through, and it + // is a real trade-off: a match past the clamp is reported as no match. A + // literal search still finds it, because literal modes are linear and are + // handed the whole value. + expect(searchJson(long, { query: 'needle', mode: 'regex' }).results).toEqual([]); + expect(searchJson(long, { query: 'needle' }).results.map((r) => r.path)).toEqual(['note']); + }); + + it('still matches inside a value that fits under the clamp', () => { + const short = { note: `${'a'.repeat(REGEX_INPUT_LIMIT - 10)}needle` }; + expect( + searchJson(short, { query: 'needle$', mode: 'regex' }).results.map((r) => r.path) + ).toEqual(['note']); + }); + + it('reads the clock before every value once a user regex is in play', () => { + const wide = Object.fromEntries( + Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`]) + ); + // A clock that jumps past the budget on its second reading. It is only ever + // read twice if the walker asks per value; the sampled interval would not + // ask again for another thousand nodes, and a single pathological match + // would then be paid ninety-nine more times. + const jumpAfterFirstRead = () => { + let reads = 0; + return () => (reads++ === 0 ? 0 : 10_000); + }; + + const guarded = searchJson(wide, { + query: 'value', + mode: 'regex', + timeBudgetMs: 100, + now: jumpAfterFirstRead(), + }); + expect(guarded.budgetExhausted).toBe(true); + expect(guarded.visited).toBe(1); + + // Literal modes keep the sampled interval: `now()` per node would be pure + // overhead there, so this document finishes despite the same clock. + const literal = searchJson(wide, { + query: 'value', + timeBudgetMs: 100, + now: jumpAfterFirstRead(), + }); + expect(literal.budgetExhausted).toBe(false); + expect(literal.results).toHaveLength(100); + }); +}); + +describe('searchJson - result cap', () => { + const wide = Object.fromEntries(Array.from({ length: 40 }, (_, i) => [`key${i}`, 'value'])); + + it('caps materialised results but still reports the true total', () => { + const outcome = searchJson(wide, { query: 'key', resultLimit: 10 }); + expect(outcome.results).toHaveLength(10); + expect(outcome.totalMatches).toBe(40); + expect(outcome.truncated).toBe(true); + }); + + it('does not flag truncation when everything fits', () => { + const outcome = searchJson(wide, { query: 'key', resultLimit: 100 }); + expect(outcome.results).toHaveLength(40); + expect(outcome.truncated).toBe(false); + }); + + it('stops walking once the node budget is spent', () => { + const outcome = searchJson(wide, { query: 'key', nodeBudget: 5 }); + expect(outcome.budgetExhausted).toBe(true); + expect(outcome.totalMatches).toBeLessThan(40); + expect(outcome.visited).toBeLessThanOrEqual(6); + }); +}); + +describe('searchJson - filters', () => { + const doc = { + label: 'label text', + count: 42, + flag: true, + empty: null, + nested: { label: 'inner label' }, + list: ['label item'], + }; + + it('type filters hide entries without stopping the walk', () => { + // Before the fix, unchecking "objects" returned nothing at all because the + // document root is itself an object. + const outcome = searchJson(doc, { + query: 'label', + filters: { ...DEFAULT_TYPE_FILTERS, objects: false }, + }); + const paths = outcome.results.map((r) => r.path); + expect(paths).toContain('nested.label'); + expect(paths).not.toContain('nested'); + }); + + it('honours the numbers, booleans and nulls filters', () => { + expect(searchJson(doc, { query: '42' }).results.map((r) => r.path)).toEqual(['count']); + expect( + searchJson(doc, { query: '42', filters: { ...DEFAULT_TYPE_FILTERS, numbers: false } }).results + ).toEqual([]); + expect(searchJson(doc, { query: 'true' }).results.map((r) => r.path)).toEqual(['flag']); + expect(searchJson(doc, { query: 'null' }).results.map((r) => r.path)).toEqual(['empty']); + }); + + it('limits the search target to keys or values', () => { + const keysOnly = searchJson(doc, { query: 'label', target: 'keys' }); + expect(keysOnly.results.every((r) => r.type === 'key')).toBe(true); + expect(keysOnly.results.map((r) => r.path)).toEqual(['label', 'nested.label']); + + const valuesOnly = searchJson(doc, { query: 'label', target: 'values' }); + expect(valuesOnly.results.every((r) => r.type === 'value')).toBe(true); + }); + + it('reports the data type even when only keys are searched', () => { + const outcome = searchJson({ label: [1, 2] }, { query: 'label', target: 'keys' }); + expect(outcome.results[0].dataType).toBe('array'); + }); + + it('applies string length filters to value matches only', () => { + const lengths = { note: 'note', other: 'a much longer note value' }; + const outcome = searchJson(lengths, { query: 'note', minLength: 10 }); + // The short value is excluded from value matching but its key still counts; + // the long value matches by value only. + expect(outcome.results.map((r) => [r.path, r.type])).toEqual([ + ['note', 'key'], + ['other', 'value'], + ]); + expect( + searchJson(lengths, { query: 'note', maxLength: 10 }).results.map((r) => [r.path, r.type]) + ).toEqual([['note', 'both']]); + }); + + it('stops descending past maxDepth', () => { + const deep = { a: { label: 1, b: { label: 2 } } }; + const outcome = searchJson(deep, { query: 'label', maxDepth: 1 }); + expect(outcome.results.map((r) => r.path)).toEqual(['a.label']); + }); +}); + +describe('analyzeJsonStructure - word frequency', () => { + it('counts prototype-named words as numbers, not inherited members', () => { + // Built through JSON.parse: a `__proto__` object literal sets the prototype + // instead of creating an own property, which would make this vacuous. + const doc = JSON.parse( + '{"constructor": "constructor toString", "__proto__": "hasOwnProperty", "x": "valueOf"}' + ); + expect(Object.keys(doc)).toContain('__proto__'); + + const { wordFrequencies } = analyzeJsonStructure(doc); + const byWord = new Map(wordFrequencies.map((w) => [w.word, w])); + + expect(byWord.get('constructor')?.count).toBe(2); + expect(byWord.get('tostring')?.count).toBe(1); + expect(byWord.get('hasownproperty')?.count).toBe(1); + expect(byWord.get('__proto__')).toBeUndefined(); // underscores are separators + for (const word of wordFrequencies) { + expect(Number.isFinite(word.percentage)).toBe(true); + expect(typeof word.count).toBe('number'); + } + }); + + it('produces percentages that sum to 100 across all tracked words', () => { + const { wordFrequencies } = analyzeJsonStructure({ alpha: 'beta gamma', delta: 'beta' }); + const total = wordFrequencies.reduce((sum, w) => sum + w.percentage, 0); + expect(total).toBeCloseTo(100, 5); + }); + + it('tokenises non-ASCII scripts', () => { + const doc = { + es: 'año pequeño canción', + hi: 'नमस्ते दुनिया', + ta: 'வணக்கம் உலகம்', + }; + const words = analyzeJsonStructure(doc).wordFrequencies.map((w) => w.word); + // The ASCII-only tokeniser produced an empty list for every one of these. + expect(words).toContain('año'); + expect(words).toContain('pequeño'); + expect(words).toContain('canción'); + expect(words).toContain('नमस्ते'); + expect(words).toContain('दुनिया'); + expect(words).toContain('வணக்கம்'); + }); + + it('keeps space-less scripts as a single token (deliberate simplification)', () => { + // CJK has no word separators; segmenting it properly is out of scope, so a + // sentence stays one token rather than vanishing as it did before. + expect(tokenizeWords('你好世界')).toEqual(['你好世界']); + }); + + it('ignores tokens shorter than three characters', () => { + expect(tokenizeWords('a bb ccc dddd')).toEqual(['ccc', 'dddd']); + }); +}); + +describe('analyzeJsonStructure - structure', () => { + it('reports keys, values, depth and type counts', () => { + const analysis = analyzeJsonStructure({ a: 1, b: { c: 'x', d: [true, null] } }); + expect(analysis.totalKeys).toBe(4); + expect(analysis.maxDepth).toBe(3); + expect(analysis.dataTypes.object).toBe(2); + expect(analysis.dataTypes.array).toBe(1); + expect(analysis.dataTypes.string).toBe(1); + }); + + it('caps the stored path sample and says so', () => { + const wide = Object.fromEntries(Array.from({ length: 50 }, (_, i) => [`k${i}`, i])); + const analysis = analyzeJsonStructure(wide, { pathLimit: 10 }); + expect(analysis.paths).toHaveLength(10); + expect(analysis.pathsTruncated).toBe(true); + expect(analysis.totalKeys).toBe(50); // counting is unaffected by the cap + }); + + it('caps the number of distinct tracked words', () => { + const doc = Object.fromEntries(Array.from({ length: 50 }, (_, i) => [`word${i}aaa`, 1])); + const analysis = analyzeJsonStructure(doc, { wordLimit: 5, topWords: 100 }); + expect(analysis.wordFrequencies).toHaveLength(5); + }); +}); + +describe('buildSuggestions', () => { + const paths = Array.from({ length: 100 }, (_, i) => `user.name${i}`); + + it('returns at most five path suggestions and stops scanning early', () => { + const suggestions = buildSuggestions('name', paths, []); + expect(suggestions).toHaveLength(5); + expect(suggestions[0]).toBe('user.name0'); + }); + + it('merges word suggestions and de-duplicates', () => { + const suggestions = buildSuggestions( + 'nam', + ['name'], + [ + { word: 'name', count: 3, percentage: 1 }, + { word: 'named', count: 1, percentage: 1 }, + ] + ); + expect(suggestions).toEqual(['name', 'named']); + }); + + it('returns nothing for an empty query', () => { + expect(buildSuggestions('', paths, [])).toEqual([]); + }); +}); + +describe('highlightSegments', () => { + it('highlights every occurrence in contains mode', () => { + expect(highlightSegments('abcabc', 'bc', 'contains', false)).toEqual([ + { text: 'a', match: false }, + { text: 'bc', match: true }, + { text: 'a', match: false }, + { text: 'bc', match: true }, + ]); + }); + + it('respects caseSensitive', () => { + expect(highlightSegments('Hello hello', 'hello', 'contains', true)).toEqual([ + { text: 'Hello ', match: false }, + { text: 'hello', match: true }, + ]); + expect( + highlightSegments('Hello hello', 'hello', 'contains', false).filter((s) => s.match) + ).toHaveLength(2); + }); + + it('highlights nothing partial in exact mode', () => { + expect(highlightSegments('abcdef', 'abc', 'exact', false)).toEqual([ + { text: 'abcdef', match: false }, + ]); + expect(highlightSegments('abc', 'abc', 'exact', false)).toEqual([{ text: 'abc', match: true }]); + }); + + it('anchors starts and ends modes', () => { + expect(highlightSegments('abcdef', 'abc', 'starts', false)).toEqual([ + { text: 'abc', match: true }, + { text: 'def', match: false }, + ]); + expect(highlightSegments('abcdef', 'def', 'ends', false)).toEqual([ + { text: 'abc', match: false }, + { text: 'def', match: true }, + ]); + expect(highlightSegments('abcdef', 'abc', 'ends', false)).toEqual([ + { text: 'abcdef', match: false }, + ]); + }); + + it('leaves regex mode untouched', () => { + expect(highlightSegments('abc', 'a.c', 'regex', false)).toEqual([ + { text: 'abc', match: false }, + ]); + }); +}); + +/* ------------------------------------------------------------------ * + * Depth safety + * ------------------------------------------------------------------ */ + +describe('depth guard', () => { + it('searches a 100,000-level document instead of throwing RangeError', () => { + const deep = { shallow: 'needle', deep: nestDeeply(100_000, 'object') }; + const outcome = searchJson(deep, { query: 'needle' }); + + expect(outcome.depthExceeded).toBe(true); + // The rest of the document was still searched in full. + expect(outcome.results.map((r) => r.path)).toEqual(['shallow']); + expect(outcome.budgetExhausted).toBe(false); + }); + + it('analyses a 100,000-level document instead of throwing RangeError', () => { + const analysis = analyzeJsonStructure({ top: 'value', deep: nestDeeply(100_000, 'array') }); + + expect(analysis.depthExceeded).toBe(true); + expect(analysis.maxDepth).toBeLessThanOrEqual(MAX_JSON_DEPTH); + expect(analysis.paths).toContain('top'); + }); + + it('reports nothing skipped for a document that fits', () => { + const atLimit = nestDeeply(MAX_JSON_DEPTH - 1, 'object'); + expect(searchJson({ a: atLimit }, { query: '1' }).depthExceeded).toBe(false); + expect(analyzeJsonStructure({ a: atLimit }).depthExceeded).toBe(false); + }); + + it('leaves wide-but-shallow documents alone', () => { + const wide = Array.from({ length: 5_000 }, (_, i) => ({ name: `row-${i}` })); + const outcome = searchJson(wide, { query: 'row-4999' }); + expect(outcome.depthExceeded).toBe(false); + expect(outcome.results).toHaveLength(1); + expect(analyzeJsonStructure(wide).depthExceeded).toBe(false); + }); + + it('does not change results for ordinary data', () => { + const outcome = searchJson(sample, { query: 'name' }); + expect(outcome.depthExceeded).toBe(false); + expect(outcome.results.map((r) => r.path).sort()).toEqual( + ['admin.name', 'user.address.name', 'user.name', 'user.nickname'].sort() + ); + }); +}); + +describe('depth guard - the boundary both walkers stop at', () => { + // `nestDeeply(n)` wraps the scalar `1` in n containers, so the containers + // occupy levels 0 … n-1 and the scalar sits at level n. The ceiling is about + // the level a container's *members* land on, which is what makes both walkers + // agree: the search never recurses into a scalar, so a guard on entry would + // let it report a match one level past the limit that the analysis pass, which + // does recurse into scalars, called too deep. + const walk = (levels: number) => { + const doc = nestDeeply(levels, 'object'); + return { search: searchJson(doc, { query: '1' }), analysis: analyzeJsonStructure(doc) }; + }; + + it('one level short of the limit: everything searched', () => { + const { search, analysis } = walk(MAX_JSON_DEPTH - 1); + expect(search.depthExceeded).toBe(false); + expect(search.results).toHaveLength(1); + expect(analysis.depthExceeded).toBe(false); + expect(analysis.maxDepth).toBe(MAX_JSON_DEPTH - 1); + }); + + it('exactly at the limit: still everything searched', () => { + const { search, analysis } = walk(MAX_JSON_DEPTH); + expect(search.depthExceeded).toBe(false); + expect(search.results).toHaveLength(1); + expect(analysis.depthExceeded).toBe(false); + expect(analysis.maxDepth).toBe(MAX_JSON_DEPTH); + }); + + it('one level past the limit: both stop, and both say so', () => { + const { search, analysis } = walk(MAX_JSON_DEPTH + 1); + // Before the fix these disagreed on this exact document: the search + // returned results=1 with depthExceeded=false — a match one level past its + // own stated limit, reported as though nothing had been skipped — while the + // analysis returned maxDepth=512 with depthExceeded=true. + expect(search.depthExceeded).toBe(true); + expect(search.results).toEqual([]); + expect(analysis.depthExceeded).toBe(true); + expect(analysis.maxDepth).toBe(MAX_JSON_DEPTH); + }); + + it('reports no skipped branch for an empty container sitting at the limit', () => { + // The innermost container is exactly at the ceiling and has no members, so + // there is nothing below it to skip and nothing to warn about. + let doc: unknown = {}; + for (let i = 0; i < MAX_JSON_DEPTH; i++) doc = { a: doc }; + expect(searchJson(doc, { query: 'a' }).depthExceeded).toBe(false); + expect(analyzeJsonStructure(doc).depthExceeded).toBe(false); + }); +}); diff --git a/src/utils/jsonSearch.ts b/src/utils/jsonSearch.ts new file mode 100644 index 0000000..ead1f5d --- /dev/null +++ b/src/utils/jsonSearch.ts @@ -0,0 +1,875 @@ +/** + * Pure search engine behind SearchView. + * + * Everything here is side-effect free so it can be unit tested: the walker, the + * match predicate, the structure analysis, the suggestion list and the + * highlighter. The React component only owns state and rendering. + * + * ## Hostile regexes + * + * Regex mode hands a user-written pattern to the engine, and a backtracking + * engine can spend exponential time on patterns that take three seconds to + * type: `(a+)+$`, `((a+))+$`, `(a|a)*$`, `^(\w+\s?)*$`. Measured here, + * `((a+))+$` against `'a'.repeat(n) + '!'` costs 47 ms at n=20 and 2.6 s at + * n=28 — roughly 4x per two added characters, so hours by n=40. Nothing in this + * module can preempt a single `RegExp.test` call: the node budget and the time + * budget are both checked *between* matches, so once the engine is inside one + * pathological call the tab is frozen until it returns, and the user loses the + * document sitting in the editor. + * + * There is no sound in-thread defence — the real fix is a Web Worker, which is + * terminable, and is left to the performance phase (see the follow-up note on + * {@link compileUserRegex}). What is here instead is defence in depth, ordered + * cheapest first. Treat it as raising the cost of an accident, not as a + * guarantee: polynomial patterns such as `a*a*a*b` or `.*.*.*=` have no + * quantified group for the syntactic hint to see and stay cheap at the probe's + * ladder length, so they pass every layer below. They are slow, not hanging — + * the exponential families are what these layers actually stop. + * + * 1. {@link MAX_PATTERN_LENGTH} — refuse patterns nobody types by hand. + * 2. {@link NESTED_QUANTIFIER} — a cheap syntactic hint that catches the + * textbook shapes, including the deeply nested ones the probe below would be + * slowest to measure. It is a hint, not a proof: it cannot see across nested + * parentheses (`((a+))+`), across alternation (`(a|a)*`) or into brace + * quantifiers (`(a{2,})+`). + * 3. {@link probeRegexCost} — actually *runs* the compiled pattern against short + * adversarial strings of escalating length, on an escalating clock budget, + * and refuses it if the cost blows up. This is the check that catches the + * families the syntactic hint misses, and it is empirical rather than + * syntactic, so it does not have to predict the shape of the attack. + * 4. {@link REGEX_INPUT_LIMIT} — whatever gets through is only ever run against + * a few KB of each value, so no single match scales with document size. + * Note this bounds the input, NOT the cost: a polynomial pattern is still + * polynomial in that few KB. + * 5. A per-value clock check in {@link searchJson} — one slow match aborts the + * walk instead of being repeated for every remaining value. + * + * Together these make the known hazards safe and the unknown ones survivable. + * They do not make an accepted pattern *verified*: layers 2 and 3 are + * heuristics, and the messages they produce say the pattern was not run, never + * that the ones that are run are safe. + */ + +import { isWithinJsonDepth, MAX_JSON_DEPTH } from './jsonWalk'; + +export type SearchMode = 'contains' | 'exact' | 'starts' | 'ends' | 'regex'; +export type SearchTarget = 'both' | 'keys' | 'values'; + +export interface SearchResult { + type: 'key' | 'value' | 'both'; + path: string; + key: string; + value: unknown; + depth: number; + dataType: string; +} + +export interface TypeFilters { + strings: boolean; + numbers: boolean; + booleans: boolean; + objects: boolean; + arrays: boolean; + nulls: boolean; +} + +export interface SearchOptions { + query: string; + mode?: SearchMode; + target?: SearchTarget; + caseSensitive?: boolean; + /** -1 = unlimited */ + maxDepth?: number; + /** 0 = no minimum. Applies to string values only. */ + minLength?: number; + /** -1 = no maximum. Applies to string values only. */ + maxLength?: number; + pathPattern?: string; + excludePattern?: string; + filters?: TypeFilters; + /** How many results are materialised. Matches beyond this are counted, not kept. */ + resultLimit?: number; + /** Hard ceiling on visited nodes so a huge document cannot freeze the tab. */ + nodeBudget?: number; + /** Wall-clock ceiling, checked periodically. Pass Infinity to disable. */ + timeBudgetMs?: number; + /** + * Hard nesting ceiling. Not a user filter like `maxDepth` — this is the depth + * past which recursion would blow the stack, so the walk stops and says so. + */ + depthLimit?: number; + /** Injectable clock, for deterministic tests. */ + now?: () => number; +} + +export interface SearchOutcome { + /** At most `resultLimit` entries. */ + results: SearchResult[]; + /** Total matches seen. A lower bound when `budgetExhausted` is true. */ + totalMatches: number; + /** True when `totalMatches` exceeds what `results` holds. */ + truncated: boolean; + /** True when the walk stopped early: results and counts are incomplete. */ + budgetExhausted: boolean; + /** + * True when some branch was nested deeper than `depthLimit` and was left + * unsearched. Independent of `budgetExhausted`: the rest of the document was + * still searched in full. + */ + depthExceeded: boolean; + /** Number of nodes visited before the walk finished or gave up. */ + visited: number; + queryError: RegexIssue | null; + pathPatternError: RegexIssue | null; + excludePatternError: RegexIssue | null; +} + +/** Why a user-supplied pattern was refused. */ +export type RegexIssueCode = + /** Longer than {@link MAX_PATTERN_LENGTH}. */ + | 'too-long' + /** A quantified group that itself contains a quantifier, e.g. `(a+)+`. */ + | 'nested-quantifier' + /** Measured as catastrophically slow by {@link probeRegexCost}. */ + | 'too-slow' + /** The engine refused to compile it. */ + | 'invalid'; + +/** + * A refused pattern, as a code plus the numbers a message needs. + * + * Deliberately not a sentence: this module has no locale, and the twelve + * translations of each message live with the component that renders them. + */ +export interface RegexIssue { + code: RegexIssueCode; + /** The pattern's own length. Set for `too-long`. */ + length?: number; + /** The limit the pattern broke. Set for `too-long`. */ + limit?: number; + /** + * The engine's own explanation, set for `invalid`. Untranslated by nature — + * it comes from the JS runtime, not from us. + */ + detail?: string; +} + +export interface WordFrequency { + word: string; + count: number; + percentage: number; +} + +export interface StructureAnalysis { + /** Sampled key paths, capped at `pathLimit`. */ + paths: string[]; + pathsTruncated: boolean; + totalKeys: number; + totalValues: number; + /** Deepest level reached. Never exceeds `depthLimit` — see `depthExceeded`. */ + maxDepth: number; + /** True when the document nests deeper than `depthLimit`, so the counts are partial. */ + depthExceeded: boolean; + dataTypes: Record; + wordFrequencies: WordFrequency[]; +} + +export interface HighlightSegment { + text: string; + match: boolean; +} + +export const DEFAULT_RESULT_LIMIT = 500; +/** Generous enough that ordinary documents finish; the time budget is the real governor. */ +export const DEFAULT_NODE_BUDGET = 500_000; +export const DEFAULT_TIME_BUDGET_MS = 1_500; +/** Longer user patterns are rejected rather than compiled. */ +export const MAX_PATTERN_LENGTH = 200; +/** + * Longest string a user-supplied regex is ever run against. + * + * A match found past a few KB of a single value cannot be shown usefully — the + * result card truncates long values anyway — so the tail is worth less than the + * time a backtracking engine can spend on it. + */ +export const REGEX_INPUT_LIMIT = 4_096; +/** How long {@link probeRegexCost} may spend before calling a pattern too slow. */ +export const REGEX_PROBE_BUDGET_MS = 5; +/** Key paths kept for the sample list and the suggestion box. */ +export const DEFAULT_PATH_LIMIT = 5_000; +/** Distinct words tracked by the frequency counter. */ +export const DEFAULT_WORD_LIMIT = 20_000; +export const DEFAULT_TOP_WORDS = 30; +/** Words shorter than this are ignored by the frequency counter. */ +export const MIN_WORD_LENGTH = 3; + +const TIME_CHECK_INTERVAL = 1_024; + +/** + * Unicode-aware tokeniser. Splits on anything that is not a letter, a number or + * a combining mark, so Devanagari, Tamil, Cyrillic and accented Latin all + * tokenise correctly - marks matter, because dropping them would cut Indic + * words apart at every virama and vowel sign. Scripts written without spaces + * (Chinese, Japanese) collapse into one long token: a deliberate + * simplification, see the note in tokenizeWords' tests. + */ +const WORD_SEPARATOR = /[^\p{L}\p{N}\p{M}]+/u; + +/** + * Cheap syntactic hint: a quantified group that itself contains a quantifier, + * e.g. `(a+)+`, `(a*)*`, `([a-z]+\d)*`. + * + * A hint, not a proof. The character class cannot span nested parentheses or + * brace quantifiers, so `((a+))+`, `(a|a)*` and `(a{2,})+` all sail past it — + * which is exactly why it is the second line of defence and not the only one. + * It is kept because it is free, and because it catches the deeply nested + * shapes (`(((a+)+)+)+`) that {@link probeRegexCost} would be slowest to + * measure. See the ReDoS note in the module docs. + */ +const NESTED_QUANTIFIER = /\([^()]*[*+][^()]*\)\s*[*+]/; + +/** + * Lengths the cost probe escalates through. + * + * Escalation is what keeps the probe itself cheap: catastrophic cost grows by + * about 4x per two added characters, so the budget is spent at the first length + * that blows up, and the step that finally exceeds it costs roughly one growth + * factor more than the step before — tens of milliseconds, not seconds. Running + * straight at the longest length would hand a pathological pattern the very + * freeze this is meant to prevent. + */ +const PROBE_LENGTHS = [4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32]; +/** Appended to each probe string so an anchored pattern has to fail, not match. */ +const PROBE_SENTINELS = ['!', '@', '#', '~']; +/** Used when a pattern contains no literal characters to build a probe from. */ +const PROBE_DEFAULT_SEED = 'a'; +/** Distinct literal characters taken from the pattern to build probe strings. */ +const PROBE_MAX_SEEDS = 3; + +/** + * The literal characters a probe string should be built from. + * + * Backtracking blows up on input the pattern *almost* matches, so the probe is + * built out of the pattern's own literals: `(x+x+)+y` is only slow on a run of + * x's. Escaped characters are skipped — `\d` contributes a `d` that means + * nothing to the matcher — and `a` is the fallback for a pattern made entirely + * of classes and metacharacters, which is what `\w`, `\s` and `.` all accept. + */ +function probeSeeds(pattern: string): string[] { + const seeds = new Set(); + for (let i = 0; i < pattern.length && seeds.size < PROBE_MAX_SEEDS; i++) { + const char = pattern[i]; + if (char === '\\') { + i++; + continue; + } + if (/[\p{L}\p{N}]/u.test(char)) seeds.add(char); + } + return seeds.size > 0 ? Array.from(seeds) : [PROBE_DEFAULT_SEED]; +} + +/** + * Is this pattern catastrophically slow? + * + * Runs the compiled pattern against short adversarial strings of escalating + * length and gives up the moment the accumulated time passes + * {@link REGEX_PROBE_BUDGET_MS}. Empirical rather than syntactic, so it catches + * the whole family of ambiguous patterns without having to recognise their + * shape: measured here it refuses `((a+))+$`, `(a|a)*$`, `(a+)+$`, `(?:a+)+$`, + * `(a{2,})+$`, `(x+x+)+y`, `^(\w+\s?)*$` and `^(a|a?)+$`, while ordinary + * patterns finish the whole ladder in under 0.05 ms — a hundredfold margin + * below the budget, so a loaded machine does not start refusing real queries. + * + * A negative answer is not a safety proof: a pattern whose blowup needs input + * this probe never guesses will still be accepted. {@link REGEX_INPUT_LIMIT} and + * the per-value clock check in {@link searchJson} are what bound that case. + */ +export function probeRegexCost( + regex: RegExp, + pattern: string, + now: () => number = Date.now +): boolean { + const seeds = probeSeeds(pattern); + const sentinel = PROBE_SENTINELS.find((char) => !pattern.includes(char)) ?? ' '; + const mixed = seeds.join(''); + const startedAt = now(); + + for (const length of PROBE_LENGTHS) { + const runs = [ + seeds[0].repeat(length) + sentinel, + mixed.repeat(Math.ceil(length / mixed.length)).slice(0, length) + sentinel, + ]; + for (const run of runs) { + regex.test(run); + // Checked after every single call: the budget only means something if the + // ladder can stop between two lengths, and between two strings. + if (now() - startedAt > REGEX_PROBE_BUDGET_MS) return true; + } + } + return false; +} + +export const DEFAULT_TYPE_FILTERS: TypeFilters = { + strings: true, + numbers: true, + booleans: true, + objects: true, + arrays: true, + nulls: true, +}; + +export function dataTypeOf(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + switch (typeof value) { + case 'string': + return 'string'; + case 'number': + return 'number'; + case 'boolean': + return 'boolean'; + case 'object': + return 'object'; + default: + return 'unknown'; + } +} + +function isTypeEnabled(dataType: string, filters: TypeFilters): boolean { + switch (dataType) { + case 'string': + return filters.strings; + case 'number': + return filters.numbers; + case 'boolean': + return filters.booleans; + case 'object': + return filters.objects; + case 'array': + return filters.arrays; + case 'null': + return filters.nulls; + default: + return true; + } +} + +/** + * Compiles a user-supplied pattern, refusing shapes that are cheap to type and + * expensive to run. Never uses the `g` flag: a global regex carries `lastIndex` + * between `.test()` calls, so a compiled-once matcher would alternate between + * matching and not matching the same string. + * + * The checks run cheapest first — length, then the syntactic hint, then + * compilation, then the cost probe, which is the only one that has to run the + * pattern. A refusal says the pattern was *not run*; acceptance is not a + * verdict that the pattern is safe. See the ReDoS note in the module docs. + * + * Follow-up: the terminable answer is to run the walk in a Web Worker, so a + * pattern that gets past every heuristic costs a `worker.terminate()` instead of + * the tab. That is a structural change and belongs to the performance phase. + */ +export function compileUserRegex( + pattern: string, + flags: string +): { regex: RegExp | null; error: RegexIssue | null } { + if (pattern.length > MAX_PATTERN_LENGTH) { + return { + regex: null, + error: { code: 'too-long', length: pattern.length, limit: MAX_PATTERN_LENGTH }, + }; + } + if (NESTED_QUANTIFIER.test(pattern)) { + return { regex: null, error: { code: 'nested-quantifier' } }; + } + + let regex: RegExp; + try { + regex = new RegExp(pattern, flags); + } catch (e) { + return { + regex: null, + error: { code: 'invalid', detail: e instanceof Error ? e.message : undefined }, + }; + } + + if (probeRegexCost(regex, pattern)) return { regex: null, error: { code: 'too-slow' } }; + return { regex, error: null }; +} + +export interface Matcher { + matches(text: string): boolean; +} + +/** + * Wraps a compiled user regex so it never sees more than + * {@link REGEX_INPUT_LIMIT} characters. + * + * Only regexes are clamped. A truncated string would make `endsWith` silently + * *wrong* rather than merely incomplete, so the literal modes keep the whole + * value — they are linear anyway, and cost nothing on a long one. + */ +export function regexMatcher(regex: RegExp): Matcher { + return { + matches: (text) => + regex.test(text.length > REGEX_INPUT_LIMIT ? text.slice(0, REGEX_INPUT_LIMIT) : text), + }; +} + +/** + * Builds the match predicate once per search instead of per compared value. + */ +export function compileMatcher( + query: string, + mode: SearchMode, + caseSensitive: boolean +): { matcher: Matcher | null; error: RegexIssue | null } { + if (mode === 'regex') { + const { regex, error } = compileUserRegex(query, caseSensitive ? '' : 'i'); + if (!regex) return { matcher: null, error }; + return { matcher: regexMatcher(regex), error: null }; + } + + const needle = caseSensitive ? query : query.toLowerCase(); + const prepare = (text: string) => (caseSensitive ? text : text.toLowerCase()); + + switch (mode) { + case 'exact': + return { matcher: { matches: (text) => prepare(text) === needle }, error: null }; + case 'starts': + return { matcher: { matches: (text) => prepare(text).startsWith(needle) }, error: null }; + case 'ends': + return { matcher: { matches: (text) => prepare(text).endsWith(needle) }, error: null }; + default: + return { matcher: { matches: (text) => prepare(text).includes(needle) }, error: null }; + } +} + +function emptyOutcome(overrides: Partial = {}): SearchOutcome { + return { + results: [], + totalMatches: 0, + truncated: false, + budgetExhausted: false, + depthExceeded: false, + visited: 0, + queryError: null, + pathPatternError: null, + excludePatternError: null, + ...overrides, + }; +} + +/** + * Walks the document once and collects matches. + * + * Path filtering: `pathPattern` decides whether a *candidate* is kept, tested + * against that candidate's own path - never against its container's path, which + * would make any pattern that does not also match the empty string return zero + * results. `excludePattern` is the only pattern that prunes, and it prunes the + * whole subtree so excluded branches cost nothing. + */ +export function searchJson(json: unknown, options: SearchOptions): SearchOutcome { + const { + query, + mode = 'contains', + target = 'both', + caseSensitive = false, + maxDepth = -1, + minLength = 0, + maxLength = -1, + pathPattern = '', + excludePattern = '', + filters = DEFAULT_TYPE_FILTERS, + resultLimit = DEFAULT_RESULT_LIMIT, + nodeBudget = DEFAULT_NODE_BUDGET, + timeBudgetMs = DEFAULT_TIME_BUDGET_MS, + depthLimit = MAX_JSON_DEPTH, + now = Date.now, + } = options; + + // Only an empty query short-circuits: compiling one would build a matcher + // that matches every string. A `null`, `0`, `false` or `""` document is a + // document, and the walk below simply finds nothing inside a scalar root — + // bailing out here instead would swallow the pattern errors the user needs. + if (!query.trim()) return emptyOutcome(); + + const { matcher, error: queryError } = compileMatcher(query, mode, caseSensitive); + if (!matcher) return emptyOutcome({ queryError }); + + // Path filters stay case-insensitive regardless of the query's own setting. + const path = pathPattern + ? compileUserRegex(pathPattern, 'i') + : { regex: null, error: null as RegexIssue | null }; + const exclude = excludePattern + ? compileUserRegex(excludePattern, 'i') + : { regex: null, error: null as RegexIssue | null }; + + const results: SearchResult[] = []; + let totalMatches = 0; + let visited = 0; + let budgetExhausted = false; + let depthExceeded = false; + const startedAt = now(); + + // A user regex can cost more on one value than a hundred thousand literal + // comparisons do, so the clock is read before *every* value rather than every + // few thousand: one pathological match then aborts the walk instead of being + // paid again for each of the values behind it. Literal modes keep the sampled + // interval, where `now()` per node would be pure overhead. + const timeCheckInterval = + mode === 'regex' || pathPattern || excludePattern ? 1 : TIME_CHECK_INTERVAL; + + const consumeBudget = (): boolean => { + visited++; + if (visited > nodeBudget) { + budgetExhausted = true; + return false; + } + if (visited % timeCheckInterval === 0 && now() - startedAt > timeBudgetMs) { + budgetExhausted = true; + return false; + } + return true; + }; + + const matchesValue = (value: unknown, dataType: string): boolean => { + switch (dataType) { + case 'string': { + const text = value as string; + // Length filters constrain value matches only; a key match on the same + // entry is still reported. + if (minLength > 0 && text.length < minLength) return false; + if (maxLength !== -1 && text.length > maxLength) return false; + return matcher.matches(text); + } + case 'number': + case 'boolean': + return matcher.matches(String(value)); + case 'null': + return matcher.matches('null'); + default: + // Objects and arrays are matched through their children, not by value. + return false; + } + }; + + /** + * The stack ceiling, asked once per container about the level its members + * occupy — every member of a container sits exactly one level down, scalar + * leaves included. Applied here rather than on entry to `walk` so it is the + * same test `analyzeJsonStructure` applies: this walker never recurses into a + * scalar, so an entry guard would let it report a match one level past the + * limit while the analysis pass called the identical document too deep. + * Asked only when there is something to skip, so an empty container sitting + * exactly at the limit does not report a phantom skipped branch. + */ + const membersWithinLimit = (depth: number): boolean => { + if (isWithinJsonDepth(depth + 1, depthLimit)) return true; + depthExceeded = true; + return false; + }; + + const walk = (node: unknown, nodePath: string, depth: number) => { + if (budgetExhausted) return; + if (maxDepth !== -1 && depth > maxDepth) return; + if (node === null || typeof node !== 'object') return; + + if (Array.isArray(node)) { + if (node.length > 0 && !membersWithinLimit(depth)) return; + for (let i = 0; i < node.length; i++) { + if (!consumeBudget()) return; + const childPath = `${nodePath}[${i}]`; + if (exclude.regex?.test(childPath)) continue; + walk(node[i], childPath, depth + 1); + } + return; + } + + const entries = Object.entries(node); + if (entries.length > 0 && !membersWithinLimit(depth)) return; + + for (const [key, value] of entries) { + if (!consumeBudget()) return; + const currentPath = nodePath ? `${nodePath}.${key}` : key; + + // Exclusion prunes early so excluded subtrees are never walked. + if (exclude.regex?.test(currentPath)) continue; + + const dataType = dataTypeOf(value); + const eligible = + isTypeEnabled(dataType, filters) && (!path.regex || path.regex.test(currentPath)); + + if (eligible) { + const keyMatch = target !== 'values' && matcher.matches(key); + const valueMatch = target !== 'keys' && matchesValue(value, dataType); + + if (keyMatch || valueMatch) { + totalMatches++; + if (results.length < resultLimit) { + results.push({ + type: keyMatch && valueMatch ? 'both' : keyMatch ? 'key' : 'value', + path: currentPath, + key, + value, + depth, + dataType, + }); + } + } + } + + // Type filters hide entries from the result list; they never stop the + // walk, otherwise unchecking "Objects" would empty every search because + // the document root is itself an object. + if (value && typeof value === 'object') walk(value, currentPath, depth + 1); + } + }; + + walk(json, '', 0); + + return { + results, + totalMatches, + truncated: totalMatches > results.length, + budgetExhausted, + depthExceeded, + visited, + queryError: null, + pathPatternError: path.error, + excludePatternError: exclude.error, + }; +} + +/** + * Unicode-aware word split. Returns lowercase tokens of at least + * `MIN_WORD_LENGTH` characters. + */ +export function tokenizeWords(text: string): string[] { + const tokens: string[] = []; + for (const raw of text.split(WORD_SEPARATOR)) { + if (raw.length >= MIN_WORD_LENGTH) tokens.push(raw.toLowerCase()); + } + return tokens; +} + +export interface AnalyzeOptions { + pathLimit?: number; + wordLimit?: number; + topWords?: number; + /** Hard nesting ceiling; deeper branches are left unvisited. */ + depthLimit?: number; +} + +/** + * Collects the insight-panel data in a single pass. + * + * Word counts live in a Map: a plain object inherits `constructor`, `toString` + * and friends, so `counts[word] || 0` would read `Object` for the word + * "constructor" and turn every later total into a string concatenation, which + * made every percentage NaN. + */ +export function analyzeJsonStructure( + json: unknown, + { + pathLimit = DEFAULT_PATH_LIMIT, + wordLimit = DEFAULT_WORD_LIMIT, + topWords = DEFAULT_TOP_WORDS, + depthLimit = MAX_JSON_DEPTH, + }: AnalyzeOptions = {} +): StructureAnalysis { + const paths: string[] = []; + let pathsTruncated = false; + const words = new Map(); + const dataTypes: Record = {}; + let totalKeys = 0; + let totalValues = 0; + let maxDepth = 0; + let depthExceeded = false; + + const countWords = (text: string) => { + for (const word of tokenizeWords(text)) { + const current = words.get(word); + if (current === undefined) { + if (words.size >= wordLimit) continue; + words.set(word, 1); + } else { + words.set(word, current + 1); + } + } + }; + + const bump = (type: string) => { + dataTypes[type] = (dataTypes[type] || 0) + 1; + }; + + // The same ceiling `searchJson` applies, asked at the same point: once per + // container, about the level its members occupy, and only when the container + // has members to skip. + const membersWithinLimit = (depth: number): boolean => { + if (isWithinJsonDepth(depth + 1, depthLimit)) return true; + depthExceeded = true; + return false; + }; + + const analyze = (node: unknown, nodePath: string, depth: number) => { + if (depth > maxDepth) maxDepth = depth; + + if (Array.isArray(node)) { + bump('array'); + if (node.length > 0 && !membersWithinLimit(depth)) return; + for (let i = 0; i < node.length; i++) { + analyze(node[i], `${nodePath}[${i}]`, depth + 1); + } + return; + } + + if (!node || typeof node !== 'object') return; + + bump('object'); + const entries = Object.entries(node); + if (entries.length > 0 && !membersWithinLimit(depth)) return; + + for (const [key, value] of entries) { + totalKeys++; + const currentPath = nodePath ? `${nodePath}.${key}` : key; + if (paths.length < pathLimit) { + paths.push(currentPath); + } else { + pathsTruncated = true; + } + + countWords(key); + + const dataType = dataTypeOf(value); + if (dataType === 'string') { + totalValues++; + bump('string'); + countWords(value as string); + } else if (dataType === 'number' || dataType === 'boolean' || dataType === 'null') { + totalValues++; + bump(dataType); + } + + analyze(value, currentPath, depth + 1); + } + }; + + analyze(json, '', 0); + + let totalWords = 0; + for (const count of words.values()) totalWords += count; + + const wordFrequencies: WordFrequency[] = Array.from(words, ([word, count]) => ({ + word, + count, + percentage: totalWords > 0 ? (count / totalWords) * 100 : 0, + })) + .sort((a, b) => b.count - a.count) + .slice(0, topWords); + + return { + paths, + pathsTruncated, + totalKeys, + totalValues, + maxDepth, + depthExceeded, + dataTypes, + wordFrequencies, + }; +} + +/** + * Autocomplete list. Stops scanning as soon as enough matches are found so a + * long path sample does not cost a full scan on every keystroke. + */ +export function buildSuggestions( + query: string, + paths: string[], + words: WordFrequency[], + { perSource = 5, limit = 8 }: { perSource?: number; limit?: number } = {} +): string[] { + if (!query) return []; + const needle = query.toLowerCase(); + const suggestions = new Set(); + + let found = 0; + for (const path of paths) { + if (found >= perSource) break; + if (path.toLowerCase().includes(needle)) { + suggestions.add(path); + found++; + } + } + + found = 0; + for (const word of words) { + if (found >= perSource) break; + if (word.word.startsWith(needle)) { + suggestions.add(word.word); + found++; + } + } + + return Array.from(suggestions).slice(0, limit); +} + +/** + * Splits `text` into matched / unmatched segments for rendering, honouring both + * the search mode and case sensitivity instead of highlighting every + * case-insensitive substring. + */ +export function highlightSegments( + text: string, + query: string, + mode: SearchMode, + caseSensitive: boolean +): HighlightSegment[] { + // Regex mode has no single literal to highlight. + if (!query || mode === 'regex') return [{ text, match: false }]; + + const haystack = caseSensitive ? text : text.toLowerCase(); + const needle = caseSensitive ? query : query.toLowerCase(); + + if (mode === 'exact') { + return haystack === needle ? [{ text, match: true }] : [{ text, match: false }]; + } + + if (mode === 'starts') { + if (!haystack.startsWith(needle)) return [{ text, match: false }]; + const head = text.slice(0, needle.length); + const tail = text.slice(needle.length); + return tail + ? [ + { text: head, match: true }, + { text: tail, match: false }, + ] + : [{ text: head, match: true }]; + } + + if (mode === 'ends') { + if (!haystack.endsWith(needle)) return [{ text, match: false }]; + const cut = text.length - needle.length; + const head = text.slice(0, cut); + const tail = text.slice(cut); + return head + ? [ + { text: head, match: false }, + { text: tail, match: true }, + ] + : [{ text: tail, match: true }]; + } + + const segments: HighlightSegment[] = []; + let cursor = 0; + let index = haystack.indexOf(needle); + while (index !== -1) { + if (index > cursor) segments.push({ text: text.slice(cursor, index), match: false }); + segments.push({ text: text.slice(index, index + needle.length), match: true }); + cursor = index + needle.length; + index = haystack.indexOf(needle, cursor); + } + if (cursor < text.length) segments.push({ text: text.slice(cursor), match: false }); + return segments.length > 0 ? segments : [{ text, match: false }]; +} diff --git a/src/utils/jsonStats.test.ts b/src/utils/jsonStats.test.ts new file mode 100644 index 0000000..3bc22d7 --- /dev/null +++ b/src/utils/jsonStats.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'vitest'; +import { analyzeJSON, measureShape } from './jsonStats'; +import { MAX_JSON_DEPTH, nestDeeply, runDepthGuarded } from './jsonWalk'; + +describe('analyzeJSON', () => { + it('counts keys, values, types and nesting in one pass', () => { + const stats = analyzeJSON({ name: 'ada', tags: ['a', 'b'], meta: { ok: true, note: null } }); + + expect(stats.totalKeys).toBe(5); + expect(stats.maxDepth).toBe(2); + expect(stats.typeDistribution.get('string')).toBe(3); + expect(stats.typeDistribution.get('object')).toBe(2); + expect(stats.typeDistribution.get('array')).toBe(1); + expect(stats.typeDistribution.get('boolean')).toBe(1); + expect(stats.typeDistribution.get('null')).toBe(1); + expect(stats.keyAnalysis.uniqueKeys).toEqual(new Set(['name', 'tags', 'meta', 'ok', 'note'])); + expect(stats.keyAnalysis.longestKey).toBe('name'); + }); + + it('measures array lengths', () => { + const stats = analyzeJSON({ a: [1, 2, 3], b: [], c: [1] }); + expect(stats.arrayStats).toEqual({ count: 3, minLength: 0, maxLength: 3, sumLength: 4 }); + }); + + it('records how many values sit at each level', () => { + const stats = analyzeJSON({ a: { b: 1 }, c: 2 }); + expect(Array.from(stats.depthMap.entries()).sort()).toEqual([ + [0, 1], + [1, 2], + [2, 1], + ]); + }); +}); + +describe('analyzeJSON - depth guard', () => { + it('reports a 100,000-level document instead of throwing RangeError', () => { + // `JSON.parse` accepts this document; recursion without a guard does not. + const deep = nestDeeply(100_000, 'array'); + expect(runDepthGuarded(() => analyzeJSON(deep))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + }); + + it('analyses a document that sits exactly on the limit', () => { + const atLimit = nestDeeply(MAX_JSON_DEPTH, 'object'); + const result = runDepthGuarded(() => analyzeJSON(atLimit)); + expect(result.ok).toBe(true); + expect(result.ok && result.value.maxDepth).toBe(MAX_JSON_DEPTH); + }); + + it('leaves wide-but-shallow documents alone', () => { + // The index-as-depth hazard would reject this at the 513th element. + const wide = Array.from({ length: MAX_JSON_DEPTH * 3 }, (_, i) => ({ id: i })); + const stats = analyzeJSON(wide); + expect(stats.totalKeys).toBe(MAX_JSON_DEPTH * 3); + expect(stats.maxDepth).toBe(2); + }); + + it('does not change results for ordinary data', () => { + const guarded = analyzeJSON({ a: { b: [1, 'two', null] } }); + expect(guarded.totalKeys).toBe(2); + expect(guarded.totalValues).toBe(6); + expect(guarded.maxDepth).toBe(3); + }); +}); + +describe('measureShape', () => { + it('counts every value and the deepest level', () => { + expect(measureShape({ a: { b: [1, 'two', null] } })).toEqual({ nodes: 6, depth: 3 }); + expect(measureShape([])).toEqual({ nodes: 1, depth: 0 }); + expect(measureShape({})).toEqual({ nodes: 1, depth: 0 }); + }); + + it('counts falsy scalars as the documents they are', () => { + // `0`, `false`, `""` and `null` are one-node documents, not empty ones. + for (const document of [0, false, '', null]) { + expect(measureShape(document)).toEqual({ nodes: 1, depth: 0 }); + } + }); + + it('agrees with analyzeJSON on both numbers', () => { + // The two walks are kept separate for cost, not for behaviour: analyzeJSON + // allocates a key Set and frequency Map that the status bar has no use for. + // This is the assertion that stops them drifting apart. + const documents: unknown[] = [ + null, + 0, + false, + '', + { a: { b: [1, 'two', null] } }, + [[[{ x: [1, 2, 3] }]]], + { + users: [ + { id: 1, tags: ['a'] }, + { id: 2, tags: [] }, + ], + meta: { ok: true }, + }, + Array.from({ length: 50 }, (_, i) => ({ id: i, nested: { deep: [i] } })), + ]; + + for (const document of documents) { + const stats = analyzeJSON(document); + expect(measureShape(document)).toEqual({ + nodes: stats.totalValues, + depth: stats.maxDepth, + }); + } + }); + + it('leaves wide-but-shallow documents alone', () => { + const wide = Array.from({ length: MAX_JSON_DEPTH * 3 }, (_, i) => ({ id: i })); + expect(measureShape(wide)).toEqual({ nodes: MAX_JSON_DEPTH * 6 + 1, depth: 2 }); + }); + + it('measures a document that sits exactly on the limit', () => { + const atLimit = nestDeeply(MAX_JSON_DEPTH, 'object'); + const result = runDepthGuarded(() => measureShape(atLimit)); + expect(result.ok).toBe(true); + expect(result.ok && result.value.depth).toBe(MAX_JSON_DEPTH); + }); + + it('reports a 100,000-level document instead of throwing RangeError', () => { + // The status bar renders outside every ErrorBoundary, so before the guard + // this walk unmounted the whole app — taking the user's editor content with + // it — at somewhere north of 5,000 levels. Now it declines at 512 and the + // status bar simply omits the two metrics. + const deep = nestDeeply(100_000, 'array'); + expect(runDepthGuarded(() => measureShape(deep))).toEqual({ + ok: false, + reason: 'too-deep', + limit: MAX_JSON_DEPTH, + }); + }); + + it('declines one level past the limit and accepts one level short of it', () => { + expect(runDepthGuarded(() => measureShape(nestDeeply(MAX_JSON_DEPTH + 1, 'array'))).ok).toBe( + false + ); + expect(runDepthGuarded(() => measureShape(nestDeeply(MAX_JSON_DEPTH - 1, 'array'))).ok).toBe( + true + ); + }); +}); diff --git a/src/utils/jsonStats.ts b/src/utils/jsonStats.ts new file mode 100644 index 0000000..fd41b87 --- /dev/null +++ b/src/utils/jsonStats.ts @@ -0,0 +1,110 @@ +/** + * The document walk behind the Stats view. + * + * Extracted from `StatsView.tsx` unchanged so it can be unit tested: the vitest + * environment is `node`, so nothing inside a component is reachable from a test, + * and this is one of the recursive walkers a deeply nested document kills. + * + * The component keeps only the derivation and the rendering. + */ + +import { assertJsonDepth } from './jsonWalk'; + +/** How big and how deep a document is — the two numbers the status bar shows. */ +export interface JsonShape { + /** Every value in the document, containers included. */ + nodes: number; + /** Deepest nesting level reached. The root is level 0. */ + depth: number; +} + +/** + * Count values and measure nesting in one pass. + * + * Deliberately a leaner walk than {@link analyzeJSON}, which produces the same + * two numbers as `totalValues` and `maxDepth`. The status bar runs on every + * document the user opens, whichever tab is showing, and `analyzeJSON` allocates + * a `Set` of every distinct key plus a frequency `Map` beside it — storage sized + * to the document, for two numbers that need none of it. `jsonStats.test.ts` + * asserts the two walks agree, so keeping them separate cannot let them drift. + * + * Throws `JsonDepthLimitError` past `MAX_JSON_DEPTH`. StatusBar runs it inside + * `runDepthGuarded` and drops the two metrics rather than crashing — it renders + * outside every error boundary, so an unguarded walk there takes the whole app + * down and the user's document with it. + */ +export function measureShape( + data: any, + depth = 0, + shape: JsonShape = { nodes: 0, depth: 0 } +): JsonShape { + assertJsonDepth(depth); + shape.nodes++; + if (depth > shape.depth) shape.depth = depth; + + if (data !== null && typeof data === 'object') { + const children = Array.isArray(data) ? data : Object.values(data); + for (const child of children) measureShape(child, depth + 1, shape); + } + + return shape; +} + +export interface JSONStats { + totalKeys: number; + totalValues: number; + maxDepth: number; + typeDistribution: Map; + arrayStats: { count: number; minLength: number; maxLength: number; sumLength: number }; + keyAnalysis: { uniqueKeys: Set; keyFrequency: Map; longestKey: string }; + depthMap: Map; +} + +/** + * Count keys, values, types, array sizes and nesting in one pass. + * + * `stats` is the accumulator threaded through the recursion; callers pass only + * `data`. Throws `JsonDepthLimitError` past `MAX_JSON_DEPTH`; StatsView runs it + * inside `runDepthGuarded` and shows the "nested too deeply" panel instead. + */ +export function analyzeJSON(data: any, depth = 0, stats?: JSONStats): JSONStats { + assertJsonDepth(depth); + if (!stats) { + stats = { + totalKeys: 0, + totalValues: 0, + maxDepth: 0, + typeDistribution: new Map(), + arrayStats: { count: 0, minLength: Infinity, maxLength: 0, sumLength: 0 }, + keyAnalysis: { uniqueKeys: new Set(), keyFrequency: new Map(), longestKey: '' }, + depthMap: new Map(), + }; + } + + stats.maxDepth = Math.max(stats.maxDepth, depth); + stats.depthMap.set(depth, (stats.depthMap.get(depth) || 0) + 1); + + const type = data === null ? 'null' : Array.isArray(data) ? 'array' : typeof data; + stats.typeDistribution.set(type, (stats.typeDistribution.get(type) || 0) + 1); + stats.totalValues++; + + if (Array.isArray(data)) { + stats.arrayStats.count++; + stats.arrayStats.minLength = Math.min(stats.arrayStats.minLength, data.length); + stats.arrayStats.maxLength = Math.max(stats.arrayStats.maxLength, data.length); + stats.arrayStats.sumLength += data.length; + // Wrapped, never `data.map(analyzeJSON)`: `map` supplies the element index, + // which this signature would read as a depth. + data.forEach((item) => analyzeJSON(item, depth + 1, stats)); + } else if (typeof data === 'object' && data !== null) { + Object.entries(data).forEach(([key, value]) => { + stats!.totalKeys++; + stats!.keyAnalysis.uniqueKeys.add(key); + stats!.keyAnalysis.keyFrequency.set(key, (stats!.keyAnalysis.keyFrequency.get(key) || 0) + 1); + if (key.length > stats!.keyAnalysis.longestKey.length) stats!.keyAnalysis.longestKey = key; + analyzeJSON(value, depth + 1, stats); + }); + } + + return stats; +} diff --git a/src/utils/jsonUtils.test.ts b/src/utils/jsonUtils.test.ts index dc80109..63b3099 100644 --- a/src/utils/jsonUtils.test.ts +++ b/src/utils/jsonUtils.test.ts @@ -26,6 +26,29 @@ describe('formatJSON', () => { it('throws on unfixable input', () => { expect(() => formatJSON('{not json at all!!!')).toThrow(/Invalid JSON|JSON/i); }); + + it('formats a falsy scalar the fixer repaired', () => { + // These used to throw "Invalid JSON format": success was decided by the + // truthiness of the repaired value, and `0`, `false`, `null` and `""` are + // all falsy — and all perfectly valid JSON documents. + expect(formatJSON('0,')).toBe('0'); + expect(formatJSON('false,')).toBe('false'); + expect(formatJSON('null,')).toBe('null'); + expect(formatJSON("''")).toBe('""'); + }); + + it('formats a falsy scalar that needs no repair', () => { + expect(formatJSON('0')).toBe('0'); + expect(formatJSON('false')).toBe('false'); + expect(formatJSON('null')).toBe('null'); + expect(formatJSON('""')).toBe('""'); + }); + + it('formats a document whose only member is falsy', () => { + expect(formatJSON('{"a": null,}')).toBe(`{ + "a": null +}`); + }); }); describe('compactJSON', () => { @@ -50,6 +73,13 @@ describe('compactJSON', () => { // Pure non-JSON text that no heuristic can turn into a value. expect(() => compactJSON('@@@ not json @@@')).toThrow(); }); + + it('compacts a falsy scalar the fixer repaired', () => { + expect(compactJSON('0,')).toBe('0'); + expect(compactJSON('false,')).toBe('false'); + expect(compactJSON('null,')).toBe('null'); + expect(compactJSON("''")).toBe('""'); + }); }); describe('isValidJSON', () => { @@ -76,4 +106,13 @@ describe('isValidJSON', () => { expect(isValidJSON('not-json')).toBe(false); expect(isValidJSON('@@@')).toBe(false); }); + + it('accepts falsy scalars, repaired or not', () => { + // `result.data !== null` rejected `null,` outright and the truthiness tests + // in formatJSON/compactJSON rejected the other three. + for (const document of ['0', 'false', 'null', '""']) { + expect(isValidJSON(document), document).toBe(true); + expect(isValidJSON(`${document},`), `${document},`).toBe(true); + } + }); }); diff --git a/src/utils/jsonUtils.ts b/src/utils/jsonUtils.ts index 93f3301..f9d434d 100644 --- a/src/utils/jsonUtils.ts +++ b/src/utils/jsonUtils.ts @@ -1,5 +1,15 @@ +import { isParseSuccess } from './jsonDocument'; import { JSONFixer } from './jsonFixer'; +/** + * Whether a repair succeeded is decided by the absence of an error, never by the + * truthiness of the repaired value. `0`, `false`, `null` and `""` are all valid + * JSON documents, and testing the value reported every one of them as + * unfixable — `formatJSON('0,')` threw "Invalid JSON format" for a document the + * fixer had repaired cleanly. `JSONFixer.parseWithFixInfo` never returns data + * and an error together, so the two signals cannot disagree. + */ + export function formatJSON(jsonString: string): string { if (!jsonString || jsonString.trim() === '') { return ''; @@ -9,11 +19,11 @@ export function formatJSON(jsonString: string): string { // First try to parse directly const parsed = JSON.parse(jsonString); return JSON.stringify(parsed, null, 2); - } catch (error) { + } catch { // Try to fix common issues using JSONFixer const result = JSONFixer.parseWithFixInfo(jsonString); - if (result.data) { + if (isParseSuccess(result)) { return JSON.stringify(result.data, null, 2); } @@ -31,11 +41,11 @@ export function compactJSON(jsonString: string): string { // First try to parse directly const parsed = JSON.parse(jsonString); return JSON.stringify(parsed); - } catch (error) { + } catch { // Try to fix common issues using JSONFixer const result = JSONFixer.parseWithFixInfo(jsonString); - if (result.data) { + if (isParseSuccess(result)) { return JSON.stringify(result.data); } @@ -54,7 +64,6 @@ export function isValidJSON(jsonString: string): boolean { return true; } catch { // Try with JSONFixer - const result = JSONFixer.parseWithFixInfo(jsonString); - return result.data !== null; + return isParseSuccess(JSONFixer.parseWithFixInfo(jsonString)); } } diff --git a/src/utils/jsonWalk.test.ts b/src/utils/jsonWalk.test.ts new file mode 100644 index 0000000..7dc79b5 --- /dev/null +++ b/src/utils/jsonWalk.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it } from 'vitest'; +import { + assertJsonDepth, + exceedsJsonDepth, + isJsonDepthLimitError, + isStackOverflow, + isWithinJsonDepth, + JsonDepthLimitError, + MAX_JSON_DEPTH, + nestDeeply, + runDepthGuarded, +} from './jsonWalk'; + +/** The premise of this whole module: `JSON.parse` accepts what recursion cannot walk. */ +describe('the premise', () => { + it('JSON.parse accepts a 100,000-level document', () => { + const text = '['.repeat(100_000) + '1' + ']'.repeat(100_000); + expect(() => JSON.parse(text)).not.toThrow(); + }); + + it('plain recursion over that same document blows the stack', () => { + const deep = nestDeeply(100_000, 'array'); + const walk = (node: unknown): number => (Array.isArray(node) ? walk(node[0]) + 1 : 0); + expect(() => walk(deep)).toThrow(RangeError); + }); + + it('the limit is under every engine primitive the views depend on', () => { + // structuredClone (~2,218) is the tightest of them; see the module comment. + expect(() => structuredClone(nestDeeply(MAX_JSON_DEPTH))).not.toThrow(); + expect(() => JSON.stringify(nestDeeply(MAX_JSON_DEPTH))).not.toThrow(); + }); +}); + +describe('assertJsonDepth', () => { + it('is a no-op at and below the limit', () => { + expect(() => assertJsonDepth(0)).not.toThrow(); + expect(() => assertJsonDepth(MAX_JSON_DEPTH)).not.toThrow(); + }); + + it('throws a JsonDepthLimitError one level past the limit', () => { + expect(() => assertJsonDepth(MAX_JSON_DEPTH + 1)).toThrow(JsonDepthLimitError); + }); + + it('carries the limit it tripped on', () => { + try { + assertJsonDepth(11, 10); + expect.unreachable('should have thrown'); + } catch (error) { + expect(isJsonDepthLimitError(error)).toBe(true); + expect((error as JsonDepthLimitError).limit).toBe(10); + } + }); +}); + +describe('isWithinJsonDepth', () => { + it('mirrors assertJsonDepth without throwing', () => { + expect(isWithinJsonDepth(MAX_JSON_DEPTH)).toBe(true); + expect(isWithinJsonDepth(MAX_JSON_DEPTH + 1)).toBe(false); + expect(isWithinJsonDepth(3, 2)).toBe(false); + }); +}); + +describe('isStackOverflow', () => { + it('recognises the wording of every engine, not just V8', () => { + expect(isStackOverflow(new RangeError('Maximum call stack size exceeded'))).toBe(true); + // SpiderMonkey throws InternalError, so the constructor must not be tested. + expect(isStackOverflow(new Error('too much recursion'))).toBe(true); + }); + + it('does not claim unrelated RangeErrors', () => { + expect(isStackOverflow(new RangeError('Invalid typed array length'))).toBe(false); + expect(isStackOverflow('not an error')).toBe(false); + }); +}); + +describe('runDepthGuarded', () => { + it('passes a successful walk straight through', () => { + const result = runDepthGuarded(() => 42); + expect(result).toEqual({ ok: true, value: 42 }); + }); + + it('reports a depth overflow instead of throwing', () => { + const result = runDepthGuarded(() => { + assertJsonDepth(MAX_JSON_DEPTH + 1); + return 'unreachable'; + }); + expect(result).toEqual({ ok: false, reason: 'too-deep', limit: MAX_JSON_DEPTH }); + }); + + it('reports the limit the walk actually used, not the default', () => { + const result = runDepthGuarded(() => { + assertJsonDepth(6, 5); + return 'unreachable'; + }, 5); + expect(result).toEqual({ ok: false, reason: 'too-deep', limit: 5 }); + }); + + it('also absorbs a real stack overflow from recursion it does not own', () => { + const result = runDepthGuarded(() => JSON.stringify(nestDeeply(100_000))); + expect(result).toEqual({ ok: false, reason: 'too-deep', limit: MAX_JSON_DEPTH }); + }); + + it('re-throws anything that is not a depth problem', () => { + expect(() => + runDepthGuarded(() => { + throw new TypeError('a real bug'); + }) + ).toThrow(TypeError); + }); +}); + +describe('exceedsJsonDepth', () => { + it('answers for a document no recursive walker could survive', () => { + // The probe itself is iterative, which is the only reason this can be asked. + expect(exceedsJsonDepth(nestDeeply(100_000, 'array'))).toBe(true); + expect(exceedsJsonDepth(nestDeeply(100_000, 'object'))).toBe(true); + }); + + it('accepts a document exactly at the limit', () => { + expect(exceedsJsonDepth(nestDeeply(MAX_JSON_DEPTH))).toBe(false); + }); + + it('rejects a document one level past it', () => { + expect(exceedsJsonDepth(nestDeeply(MAX_JSON_DEPTH + 1))).toBe(true); + }); + + it('leaves wide-but-shallow documents alone', () => { + const wide = Array.from({ length: 50_000 }, (_, i) => ({ id: i, tags: ['a', 'b', 'c'] })); + expect(exceedsJsonDepth(wide)).toBe(false); + }); + + it('counts nesting, not member count', () => { + expect(exceedsJsonDepth({ a: { b: 1 } }, 2)).toBe(false); + expect(exceedsJsonDepth({ a: { b: 1 } }, 1)).toBe(true); + expect(exceedsJsonDepth({ a: { b: 1 } }, 0)).toBe(true); + }); + + it('treats scalars, empty containers and null as depth 0', () => { + expect(exceedsJsonDepth(1, 0)).toBe(false); + expect(exceedsJsonDepth(null, 0)).toBe(false); + expect(exceedsJsonDepth('x', 0)).toBe(false); + expect(exceedsJsonDepth({}, 0)).toBe(false); + expect(exceedsJsonDepth([], 0)).toBe(false); + }); + + it('finds the deep branch even when it is not the first one', () => { + const value = { shallow: 1, deep: nestDeeply(MAX_JSON_DEPTH + 1) }; + expect(exceedsJsonDepth(value)).toBe(true); + }); +}); + +describe('nestDeeply', () => { + it('builds the requested shape without recursing', () => { + expect(nestDeeply(0)).toBe(1); + expect(nestDeeply(2)).toEqual({ a: { a: 1 } }); + expect(nestDeeply(2, 'array')).toEqual([[1]]); + }); +}); diff --git a/src/utils/jsonWalk.ts b/src/utils/jsonWalk.ts new file mode 100644 index 0000000..c269ba7 --- /dev/null +++ b/src/utils/jsonWalk.ts @@ -0,0 +1,215 @@ +/** + * Depth safety for every recursive walk over a parsed JSON document. + * + * `JSON.parse` is iterative in V8, so a document like `[[[[…1…]]]]` with a + * hundred thousand brackets parses in milliseconds and hands the views a value + * that is perfectly valid and completely un-walkable: the first plain recursive + * traversal blows the stack with `RangeError: Maximum call stack size exceeded`. + * The views sit inside an `ErrorBoundary`, so the user gets an error panel and a + * message that means nothing to them. + * + * The fix here is a *guard*, not an iterative rewrite of the walkers, because an + * iterative walker would not actually make the deep case work. Measured on this + * machine (Node 24, V8 default ~1 MB stack — the same engine and roughly the + * same budget Chrome gives a page): + * + * JSON.parse 1,000,000+ levels (iterative; never the bottleneck) + * JSON.stringify ~6,182 levels (recursive, engine-internal) + * structuredClone ~2,218 levels (recursive, engine-internal) + * our own walkers ~4,096–4,863 levels (light and fat frames) + * + * `JSON.stringify` and `structuredClone` are engine primitives every view + * depends on — Stats measures the minified size, the tree editor clones before + * writing — and neither can be made iterative. So the real ceiling is set by the + * engine, not by our recursion; the honest fix is to bound the depth we accept + * and tell the user, rather than to walk deeper and die one line later. + * + * {@link MAX_JSON_DEPTH} is that bound. Below it every guard is a no-op and the + * walkers behave exactly as before. + */ + +/* ------------------------------------------------------------------ * + * The JSON value type + * ------------------------------------------------------------------ */ + +/** A JSON scalar. */ +export type JsonPrimitive = string | number | boolean | null; + +/** A JSON object. Declared as an interface so the union below can be recursive. */ +export interface JsonObject { + [key: string]: JsonValue; +} + +export type JsonArray = JsonValue[]; + +/** + * Anything `JSON.parse` can return. + * + * The views still type their `json` prop as `any`; this union exists so the + * later type-cleanup phase has one shared definition to adopt instead of twelve + * private ones. + */ +export type JsonValue = JsonPrimitive | JsonObject | JsonArray; + +/* ------------------------------------------------------------------ * + * The depth limit + * ------------------------------------------------------------------ */ + +/** + * Deepest nesting level any view will walk. The document root is level 0, its + * members are level 1, and so on. + * + * 512 is generous for real data by a wide margin — configuration files, API + * payloads, GeoJSON and lock files live at one or two dozen levels — and it + * leaves headroom against every measured ceiling in the module comment above: + * ~4.3x on `structuredClone` (the tightest, and the one the tree editor needs), + * ~8x on our own recursion and ~12x on `JSON.stringify`. That margin is + * deliberate: those numbers come from one engine on one machine, browsers on + * smaller stacks (and React's own frames sitting underneath ours) get less, and + * no equivalent measurement exists for JavaScriptCore or SpiderMonkey. + */ +export const MAX_JSON_DEPTH = 512; + +/** Thrown by {@link assertJsonDepth} when a walk goes past the limit. */ +export class JsonDepthLimitError extends Error { + readonly limit: number; + + constructor(limit: number) { + super(`JSON nesting exceeds the supported depth of ${limit} levels`); + this.name = 'JsonDepthLimitError'; + this.limit = limit; + } +} + +export function isJsonDepthLimitError(error: unknown): error is JsonDepthLimitError { + return error instanceof JsonDepthLimitError; +} + +/** + * Does this error look like a blown stack? + * + * Engines disagree on both the constructor and the wording — V8 and JSC throw + * `RangeError: Maximum call stack size exceeded`, SpiderMonkey throws + * `InternalError: too much recursion` — so this matches on the message and does + * not test the constructor. A heuristic by necessity, used only as a backstop + * for recursion we do not own (`JSON.stringify`, `structuredClone`). + */ +const STACK_OVERFLOW_MESSAGE = /maximum call stack|call stack size exceeded|too much recursion/i; + +export function isStackOverflow(error: unknown): boolean { + return error instanceof Error && STACK_OVERFLOW_MESSAGE.test(error.message); +} + +/** + * True while `depth` is still walkable. For walkers that already degrade + * gracefully (the search engine reports a partial result rather than failing), + * this is the non-throwing form of {@link assertJsonDepth}. + */ +export function isWithinJsonDepth(depth: number, limit: number = MAX_JSON_DEPTH): boolean { + return depth <= limit; +} + +/** + * Call at the top of each recursive step, passing that step's depth. A no-op for + * anything a browser can actually render; throws {@link JsonDepthLimitError} the + * moment the walk goes deeper than `limit`. + */ +export function assertJsonDepth(depth: number, limit: number = MAX_JSON_DEPTH): void { + if (depth > limit) throw new JsonDepthLimitError(limit); +} + +/* ------------------------------------------------------------------ * + * Running a guarded walk + * ------------------------------------------------------------------ */ + +export type DepthGuardedResult = + | { ok: true; value: T } + | { ok: false; reason: 'too-deep'; limit: number }; + +/** + * Run a guarded walk and turn a depth overflow into a value the caller can + * render, instead of an exception that reaches the error boundary. + * + * Also catches a genuine stack overflow, so a view survives recursion it does + * not own — `JSON.stringify` on the same document, say. Every other error is + * re-thrown untouched: a bug in a walker must still surface as a bug. + */ +export function runDepthGuarded( + run: () => T, + limit: number = MAX_JSON_DEPTH +): DepthGuardedResult { + try { + return { ok: true, value: run() }; + } catch (error) { + if (isJsonDepthLimitError(error)) { + return { ok: false, reason: 'too-deep', limit: error.limit }; + } + if (isStackOverflow(error)) { + return { ok: false, reason: 'too-deep', limit }; + } + throw error; + } +} + +/* ------------------------------------------------------------------ * + * Iterative probe + * ------------------------------------------------------------------ */ + +/** + * Is anything in this document nested deeper than `limit`? + * + * Walks with an explicit stack and never recurses, so it is safe on a document + * that no recursive walker could survive — which is the whole point: it is the + * one question that must be answerable *before* deciding whether to recurse. + * Returns as soon as the answer is known, and only ever holds containers on its + * stack, so a wide-but-shallow document costs one pass and no per-leaf storage. + * + * Callers that walk the document anyway should prefer {@link assertJsonDepth} + * inside their own recursion, which costs nothing; this is for the places where + * the recursion is not ours to instrument (React's render, `structuredClone`). + */ +export function exceedsJsonDepth(value: unknown, limit: number = MAX_JSON_DEPTH): boolean { + if (limit < 0) return true; + + const nodes: unknown[] = [value]; + const depths: number[] = [0]; + + while (nodes.length > 0) { + const node = nodes.pop(); + const depth = depths.pop() as number; + + if (depth > limit) return true; + if (node === null || typeof node !== 'object') continue; + + const children: unknown[] = Array.isArray(node) ? node : Object.values(node); + if (children.length === 0) continue; + // Any member at all sits one level down, primitive or not. + if (depth + 1 > limit) return true; + + for (const child of children) { + // Primitives can never be deeper than the container holding them, and the + // line above has already accounted for their level. + if (child !== null && typeof child === 'object') { + nodes.push(child); + depths.push(depth + 1); + } + } + } + + return false; +} + +/** + * Build a value nested `depth` levels deep, without recursing. + * + * Lives beside the guard rather than in a test file because the fixtures that + * exercise it are the same shape everywhere and building one by recursion would + * blow the stack before the test could run. + */ +export function nestDeeply(depth: number, kind: 'object' | 'array' = 'object'): JsonValue { + let value: JsonValue = 1; + for (let i = 0; i < depth; i++) { + value = kind === 'array' ? [value] : { a: value }; + } + return value; +} diff --git a/src/utils/monacoLoader.ts b/src/utils/monacoLoader.ts deleted file mode 100644 index 8eb1a97..0000000 --- a/src/utils/monacoLoader.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { loader } from '@monaco-editor/react'; - -// Configure Monaco Editor to load from CDN -export function configureMonacoLoader() { - // Check if we're in a browser environment - if (typeof window === 'undefined') return; - - // Load Monaco from the same origin (copied into /public/monaco at build time) - // rather than a third-party CDN — keeps data private and works offline. - loader.config({ - paths: { - vs: '/monaco/vs', - }, - }); -} - -// Disable workers to avoid CORS and network issues -export function configureMonacoEnvironment() { - if (typeof window === 'undefined') return; - - // Explicitly disable workers - Monaco will run in synchronous mode - // This is more reliable and avoids all worker-related errors - (window as any).MonacoEnvironment = { - // Return undefined to disable workers - getWorker: () => undefined, - // Alternative method to disable workers - getWorkerUrl: () => undefined, - }; -} - -// Initialize Monaco with workers disabled for reliability -export function initializeMonaco() { - // Configure environment to disable workers - try { - configureMonacoEnvironment(); - console.info('Monaco Editor: Running in synchronous mode (workers disabled for reliability)'); - } catch (error) { - console.warn('Monaco environment configuration warning:', error); - } - - // Configure the loader - configureMonacoLoader(); -} diff --git a/src/utils/safeStorage.test.ts b/src/utils/safeStorage.test.ts new file mode 100644 index 0000000..1485493 --- /dev/null +++ b/src/utils/safeStorage.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { readStored, removeStored, writeStored } from './safeStorage'; + +/** Replaces window.localStorage with something that throws on property access. */ +function withHostileStorage(make: () => Storage) { + const original = Object.getOwnPropertyDescriptor(globalThis, 'window'); + Object.defineProperty(globalThis, 'window', { + configurable: true, + value: { + get localStorage() { + return make(); + }, + }, + }); + return () => { + if (original) Object.defineProperty(globalThis, 'window', original); + else delete (globalThis as { window?: unknown }).window; + }; +} + +afterEach(() => { + delete (globalThis as { window?: unknown }).window; +}); + +describe('safeStorage', () => { + it('returns null on the server, where there is no window', () => { + expect(readStored('theme')).toBeNull(); + expect(writeStored('theme', 'dark')).toBe(false); + expect(removeStored('theme')).toBe(false); + }); + + it('survives storage that throws on property access (blocked site data)', () => { + const restore = withHostileStorage(() => { + throw new Error('SecurityError: The operation is insecure.'); + }); + try { + expect(() => readStored('theme')).not.toThrow(); + expect(readStored('theme')).toBeNull(); + expect(writeStored('theme', 'dark')).toBe(false); + expect(removeStored('theme')).toBe(false); + } finally { + restore(); + } + }); + + it('survives QuotaExceededError on write (Safari private browsing)', () => { + const store = new Map(); + const restore = withHostileStorage( + () => + ({ + getItem: (k: string) => store.get(k) ?? null, + setItem: () => { + throw new Error('QuotaExceededError'); + }, + removeItem: (k: string) => void store.delete(k), + }) as unknown as Storage + ); + try { + store.set('theme', 'light'); + expect(readStored('theme')).toBe('light'); + expect(writeStored('theme', 'dark')).toBe(false); + expect(readStored('theme')).toBe('light'); + } finally { + restore(); + } + }); + + it('round-trips through working storage', () => { + const store = new Map(); + const restore = withHostileStorage( + () => + ({ + getItem: (k: string) => store.get(k) ?? null, + setItem: (k: string, v: string) => void store.set(k, v), + removeItem: (k: string) => void store.delete(k), + }) as unknown as Storage + ); + try { + expect(writeStored('theme', 'dark')).toBe(true); + expect(readStored('theme')).toBe('dark'); + expect(removeStored('theme')).toBe(true); + expect(readStored('theme')).toBeNull(); + } finally { + restore(); + } + }); +}); + +describe('JSON-LD escaping', () => { + // Mirrors the transform in src/components/seo/JsonLd.tsx. Kept here because the + // component itself needs a DOM to render and vitest runs in a node environment. + const encode = (data: unknown) => JSON.stringify(data).replace(/ { + const encoded = encode({ name: '' }); + expect(encoded).not.toContain(''); + expect(encoded).not.toContain('<'); + }); + + it('decodes back to the original value', () => { + const data = { name: 'a < b ', nested: { x: '