diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 8bc1d9f405..a911e0bdfc 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -108,6 +108,7 @@ COPY --from=base /var/app/pnpm-workspace.yaml ./pnpm-workspace.yaml COPY --from=base /var/app/apps/web/package.json ./apps/web/package.json COPY --from=base /var/app/apps/web/healthCheck.js ./apps/web/healthCheck.js COPY --from=base /var/app/apps/web/next-runtime-config.js ./apps/web/next-runtime-config.js +COPY --from=base /var/app/apps/web/ssr-admission.js ./apps/web/ssr-admission.js COPY --from=base /var/app/apps/web/public ./apps/web/public COPY --from=base /var/app/apps/web/.next ./apps/web/.next COPY --from=base /var/app/node_modules ./node_modules @@ -130,5 +131,9 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD node # images, experimental flags, compress, ...). The preload hands the server the # config the build resolved, from .next/required-server-files.json, the way # Next's own standalone server.js does. See next-runtime-config.js. +# +# ssr-admission.js caps in-flight page renders per process (SSR_MAX_INFLIGHT, +# set in the stack file) and answers 503 above the cap before Next sees the +# request, so the edge fails over instead of queueing behind a stuck loop. WORKDIR /var/app/apps/web -CMD ["node", "--require", "./next-runtime-config.js", "./node_modules/next/dist/bin/next", "start"] +CMD ["node", "--require", "./next-runtime-config.js", "--require", "./ssr-admission.js", "./node_modules/next/dist/bin/next", "start"] diff --git a/apps/web/docker-compose.production.yml b/apps/web/docker-compose.production.yml index 3178da4f30..13ff5cc8d3 100644 --- a/apps/web/docker-compose.production.yml +++ b/apps/web/docker-compose.production.yml @@ -105,7 +105,17 @@ services: # Bound the V8 old-space so a pathological single render crashes (and the # replica restarts) rather than ballooning the heap unbounded. Heap working # set is ~2GiB, so 3GiB leaves headroom without clipping normal renders. - - NODE_OPTIONS=--max-old-space-size=3072 + # The semi-space (young generation) is raised from V8's default so an + # allocation-heavy render triggers fewer scavenges; costs ~128MiB of RSS + # per replica, well inside the limit above. + - NODE_OPTIONS=--max-old-space-size=3072 --max-semi-space-size=64 + # Per-process admission control (ssr-admission.js, loaded by the image + # CMD): above this many in-flight page renders the process answers 503 + # with Retry-After, which the edge worker treats as a failover signal. + # Normal load is a handful of renders in flight per process; the cap only + # bites once renders have slowed enough to pile up, which is exactly the + # state that used to end in a heap abort. Unset or 0 disables it. + - SSR_MAX_INFLIGHT=16 restart: always ports: - "3000:3000" diff --git a/apps/web/docker-compose.yml b/apps/web/docker-compose.yml index abafee0d81..de87321eec 100644 --- a/apps/web/docker-compose.yml +++ b/apps/web/docker-compose.yml @@ -85,6 +85,11 @@ services: - MATTERMOST_WS_ALLOWED_ORIGINS - THREESPEAK_EMBED_API_KEY - SEO_CRON_SECRET + # Same runtime knobs as production, so alpha exercises them first: + # semi-space size (fewer scavenges on allocation-heavy renders) and the + # per-process render cap (ssr-admission.js, loaded by the image CMD). + - NODE_OPTIONS=--max-semi-space-size=64 + - SSR_MAX_INFLIGHT=16 restart: always ports: - "3000:3000" diff --git a/apps/web/src/specs/ssr-admission.spec.ts b/apps/web/src/specs/ssr-admission.spec.ts new file mode 100644 index 0000000000..34f5cd8927 --- /dev/null +++ b/apps/web/src/specs/ssr-admission.spec.ts @@ -0,0 +1,277 @@ +// @vitest-environment node +import { afterEach, describe, expect, it } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { readFileSync } from "node:fs"; +import http from "node:http"; +import { join } from "node:path"; + +/** + * The production image starts Next with `node --require ./ssr-admission.js` + * (see the Dockerfile). These tests boot the real preload in a child process + * in front of a plain http server whose handler holds each page render open + * until told to finish, then drive it over real sockets: that is the same + * 'request' emission the preload intercepts for `next start`. + */ + +const PRELOAD = join(process.cwd(), "ssr-admission.js"); + +// The child: a server that parks /slow renders until /api/release is called +// (an /api/ path, so the preload never counts or sheds the control call itself), +// and answers everything else straight away. Prints its port on stdout. +const CHILD_SERVER = ` + const http = require("http"); + const parked = []; + const server = http.createServer((req, res) => { + const path = req.url.split("?")[0]; + if (path === "/api/release") { + const n = parked.length; + for (const r of parked.splice(0)) r.end("released"); + res.end(String(n)); + return; + } + if (path.startsWith("/slow")) { + parked.push(res); + return; + } + res.end("ok " + path); + }); + server.listen(0, "127.0.0.1", () => process.stdout.write(String(server.address().port) + "\\n")); +`; + +const children: ChildProcess[] = []; + +type Booted = { port: number; stderr: () => string }; +type Reply = { status: number; headers: http.IncomingHttpHeaders; body: string }; +type Started = { done: Promise<{ status: number; headers: http.IncomingHttpHeaders }>; abort: () => void }; + +function boot(env: NodeJS.ProcessEnv): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--require", PRELOAD, "-e", CHILD_SERVER], { + cwd: process.cwd(), + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"] + }); + children.push(child); + let err = ""; + child.stderr!.on("data", (d) => (err += String(d))); + child.stdout!.once("data", (d) => resolve({ port: Number(String(d).trim()), stderr: () => err })); + child.once("exit", (code) => reject(new Error(`child exited ${code}: ${err}`))); + }); +} + +function get(port: number, path: string, headers: Record = {}): Promise { + return new Promise((resolve, reject) => { + const req = http.get({ host: "127.0.0.1", port, path, headers }, (res) => { + let body = ""; + res.on("data", (d) => (body += String(d))); + res.on("end", () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body })); + }); + req.on("error", reject); + }); +} + +// Start a request and resolve once it is on the wire (the server has parked +// it) without waiting for the response. +function start(port: number, path: string): Started { + let settle!: (r: { status: number; headers: http.IncomingHttpHeaders }) => void; + const done = new Promise<{ status: number; headers: http.IncomingHttpHeaders }>((r) => (settle = r)); + const req = http.get({ host: "127.0.0.1", port, path }, (res) => { + res.resume(); + res.on("end", () => settle({ status: res.statusCode ?? 0, headers: res.headers })); + }); + req.on("error", () => settle({ status: 0, headers: {} })); + return { done, abort: () => req.destroy() }; +} + +const settle = (ms = 150): Promise => new Promise((r) => setTimeout(r, ms)); + +afterEach(() => { + for (const c of children.splice(0)) c.kill("SIGKILL"); +}); + +describe("ssr-admission preload", () => { + it("is inert without SSR_MAX_INFLIGHT", async () => { + const { port, stderr } = await boot({ SSR_MAX_INFLIGHT: "" }); + const a = start(port, "/slow/1"); + const b = start(port, "/slow/2"); + await settle(); + expect((await get(port, "/page")).status).toBe(200); + await get(port, "/api/release"); + expect((await a.done).status).toBe(200); + expect((await b.done).status).toBe(200); + expect(stderr()).toContain("disabled"); + }); + + it("answers 503 with Retry-After above the cap and frees the slot when a render finishes", async () => { + const { port } = await boot({ SSR_MAX_INFLIGHT: "2", SSR_SHED_RETRY_AFTER: "3" }); + const a = start(port, "/slow/1"); + const b = start(port, "/slow/2"); + await settle(); + + const shed = await get(port, "/@someone/some-post"); + expect(shed.status).toBe(503); + expect(shed.headers["retry-after"]).toBe("3"); + expect(shed.headers["cache-control"]).toBe("no-store"); + + // The parked renders were never touched by the shed. + expect((await get(port, "/api/release")).body).toBe("2"); + expect((await a.done).status).toBe(200); + expect((await b.done).status).toBe(200); + + // Slots are free again. + expect((await get(port, "/@someone/some-post")).status).toBe(200); + }); + + it("counts RSC navigations as renders too", async () => { + const { port } = await boot({ SSR_MAX_INFLIGHT: "1" }); + const a = start(port, "/slow/doc"); + await settle(); + expect((await get(port, "/trending?_rsc=abc12")).status).toBe(503); + await get(port, "/api/release"); + await a.done; + }); + + it("never sheds the named static paths, and counts everything else including file-like unknown paths", async () => { + const { port } = await boot({ SSR_MAX_INFLIGHT: "1" }); + const a = start(port, "/slow/doc"); + await settle(); + for (const path of [ + "/_next/static/chunks/app.js", + "/api/healthcheck", + "/api/mattermost/channels", + "/assets/noimage.png", + "/scripts/x.js", + "/favicon.ico", + "/manifest.json", + "/robots.txt", + "/sw.js", + "/firebase-messaging-sw.js", + "/og.jpg", + "/geo/cities.min.json", + "/dmca/dmca-accounts.json", + "/.well-known/assetlinks.json", + "/public-nodes.json", + "/apple-app-site-association", + "/llms.txt", + "/sitemap.xml", + "/sitemap/posts-1.xml", + "/assets/fonts/inter.woff2", + "/_next/static/media/inter.woff2" + ]) { + expect((await get(port, path)).status, path).toBe(200); + } + // Everything that renders on the loop is shed while the slot is held: a + // page, a dotted username, an RSS feed, the agent routes (a suffix the + // middleware appends to a permlink), and unknown file-looking paths, which + // render the not-found page (permlinks never contain a dot). + for (const path of [ + "/hot", + "/@demo.com", + "/@someone/rss.xml", + "/@someone/rss", + "/created/photography/rss.xml", + "/@someone/some-post.md", + "/@someone/some-post.json", + "/@someone/some-post.discussion.json", + "/@demo/post.png", + "/@demo/post.js", + "/@demo.com/avatar.jpg", + "/feed.xml", + "/notes.txt", + "/logo.png" + ]) { + expect((await get(port, path)).status, path).toBe(503); + } + await get(port, "/api/release"); + await a.done; + }); + + it("does not count writes against the cap", async () => { + const { port } = await boot({ SSR_MAX_INFLIGHT: "1" }); + const a = start(port, "/slow/doc"); + await settle(); + const status: number = await new Promise((resolve, reject) => { + const req = http.request({ host: "127.0.0.1", port, path: "/some-form", method: "POST" }, (res) => { + res.resume(); + res.on("end", () => resolve(res.statusCode ?? 0)); + }); + req.on("error", reject); + req.end("x"); + }); + expect(status).toBe(200); + await get(port, "/api/release"); + await a.done; + }); + + it("keeps the slot for the grace period after the client goes away, then frees it", async () => { + const { port } = await boot({ SSR_MAX_INFLIGHT: "1", SSR_ABANDONED_GRACE_MS: "400" }); + const a = start(port, "/slow/abandoned"); + await settle(); + expect((await get(port, "/page")).status).toBe(503); + a.abort(); + await settle(); + // The render is still running on the server; the slot is still held. + expect((await get(port, "/page")).status).toBe(503); + await settle(500); + expect((await get(port, "/page")).status).toBe(200); + await get(port, "/api/release"); + }); + + it("frees the slot as soon as an abandoned render finishes, before the grace period ends", async () => { + const { port } = await boot({ SSR_MAX_INFLIGHT: "1", SSR_ABANDONED_GRACE_MS: "5000" }); + const a = start(port, "/slow/abandoned"); + await settle(); + a.abort(); + await settle(); + expect((await get(port, "/page")).status).toBe(503); + await get(port, "/api/release"); + await settle(); + expect((await get(port, "/page")).status).toBe(200); + }); + + it("ignores an invalid cap or Retry-After rather than crashing or silently misreporting", async () => { + const bad = await boot({ SSR_MAX_INFLIGHT: "lots" }); + const a = start(bad.port, "/slow/1"); + await settle(); + expect((await get(bad.port, "/page")).status).toBe(200); + expect(bad.stderr()).toContain("is not a positive number"); + await get(bad.port, "/api/release"); + await a.done; + + const odd = await boot({ SSR_MAX_INFLIGHT: "1", SSR_SHED_RETRY_AFTER: "2\r\nX-Injected: 1" }); + const b = start(odd.port, "/slow/1"); + await settle(); + const shed = await get(odd.port, "/page"); + expect(shed.status).toBe(503); + expect(shed.headers["retry-after"]).toBe("1"); + expect(shed.headers["x-injected"]).toBeUndefined(); + await get(odd.port, "/api/release"); + await b.done; + }); + + it("is wired into the production image and the stack", () => { + const dockerfile = readFileSync(join(process.cwd(), "Dockerfile"), "utf8"); + expect(dockerfile).toContain("ssr-admission.js ./apps/web/ssr-admission.js"); + expect(dockerfile).toMatch(/CMD \[.*"--require", "\.\/ssr-admission\.js".*\]/); + // Under the web service specifically: a variable placed under another + // service is silent (the preload just reports itself disabled). + const webBlock = (compose: string): string => { + const lines = compose.split("\n"); + const start = lines.findIndex((l) => l === " web:"); + expect(start, "web service").toBeGreaterThan(-1); + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + if (/^ [A-Za-z_-]+:/.test(lines[i]) || /^[A-Za-z_-]+:/.test(lines[i])) { + end = i; + break; + } + } + return lines.slice(start, end).join("\n"); + }; + for (const file of ["docker-compose.production.yml", "docker-compose.yml"]) { + const web = webBlock(readFileSync(join(process.cwd(), file), "utf8")); + expect(web, file).toMatch(/^\s*- SSR_MAX_INFLIGHT=\d+$/m); + expect(web, file).toMatch(/^\s*- NODE_OPTIONS=.*--max-semi-space-size=\d+/m); + } + }); +}); diff --git a/apps/web/ssr-admission.js b/apps/web/ssr-admission.js new file mode 100644 index 0000000000..dc3dc9d3bf --- /dev/null +++ b/apps/web/ssr-admission.js @@ -0,0 +1,147 @@ +// Per-process admission control for page renders. +// +// Nothing inside the Next.js process limits how many renders it accepts at +// once; shedding happens only upstream and site-wide. When one render turns +// slow, requests pile up on that process, every one of them slows down, and +// the heap runaway behind the exit-134 history becomes reachable. This preload +// caps the number of in-flight page renders per process: above the cap a +// request is answered 503 with Retry-After before Next ever sees it, so the +// edge fails over to another origin instead of queueing behind a stuck loop. +// +// Hooks http.Server's 'request' emission, which is how Node hands a request to +// `next start`, so the check runs before any Next code. Only document and RSC +// renders are counted: static assets, files served from public/, API routes and +// the health check pass through uncounted. Disabled unless SSR_MAX_INFLIGHT is +// a positive number. +// +// Loaded with `node --require ./ssr-admission.js` (see the Dockerfile CMD). +// Runs before Next, so it must stay dependency-free. +const http = require("http"); + +const rawMax = process.env.SSR_MAX_INFLIGHT; +const max = rawMax === undefined || rawMax === "" ? 0 : Number(rawMax); + +// Retry-After must be a plain number of seconds; anything else would throw +// from setHeader at the worst possible moment, so it falls back to 1. +const rawRetryAfter = process.env.SSR_SHED_RETRY_AFTER; +const retryAfter = /^\d{1,4}$/.test(rawRetryAfter || "") ? rawRetryAfter : "1"; + +// A client that goes away mid-render (the edge worker's timeout, a navigation) +// does not end the render in a way this layer can see: Next aborts the stream +// and destroys the response (no end(), no finish), the render stops at its +// next chunk boundary, but an async server component already inside a chain +// of awaited prefetches runs on, each leg bounded by the server prefetch +// timeout (10s). There is no completion signal for that tail, so the slot is +// held for a fixed grace after the socket closes, long enough to cover several +// sequential legs, and the undercount is bounded to renders that outlive it, +// which the event-loop monitor reports as pathological anyway. +const rawGrace = process.env.SSR_ABANDONED_GRACE_MS; +const abandonedGraceMs = /^\d{1,6}$/.test(rawGrace || "") ? Number(rawGrace) : 30_000; + +// Paths that are not page renders, named precisely: the build output, API +// routes, the public/ directories, the static root files the app serves, and +// the Redis-backed sitemap routes. Anything else that reaches this process is +// work on the render loop and counts: a document, an RSC navigation, an RSS +// feed (`/@user/rss.xml` renders twenty posts), an agent route +// (`/@author/permlink.md|.json|.discussion.json`, a suffix the middleware +// appends to a permlink, renders the post), and an unknown path such as +// `/@author/post.png` (permlinks never contain a dot, so that is the +// not-found page, still a render). Hence no extension-based bypass at all: +// every static file this app serves lives under a prefix or at a root path +// listed here, so a name is the only safe test. +const PASS_PREFIXES = ["/_next/", "/api/", "/assets/", "/scripts/", "/geo/", "/dmca/", "/.well-known/", "/sitemap/"]; +const PASS_EXACT = new Set([ + "/favicon.ico", + "/manifest.json", + "/robots.txt", + "/llms.txt", + "/sitemap.xml", + "/sw.js", + "/firebase-messaging-sw.js", + "/og.jpg", + "/next.svg", + "/vercel.svg", + "/public-nodes.json", + "/apple-app-site-association" +]); + +function isRender(req) { + if (req.method !== "GET" && req.method !== "HEAD") return false; + const url = req.url || "/"; + const q = url.indexOf("?"); + const path = q === -1 ? url : url.slice(0, q); + if (PASS_EXACT.has(path)) return false; + for (const prefix of PASS_PREFIXES) { + if (path.startsWith(prefix)) return false; + } + return true; +} + +const state = { max, inflight: 0, shed: 0, abandoned: 0 }; +// Read by the event-loop monitor's log lines; never written from outside. +globalThis.__ecencySsrAdmission = state; + +if (Number.isFinite(max) && max > 0) { + const originalEmit = http.Server.prototype.emit; + http.Server.prototype.emit = function emit(event, req, res) { + if (event === "request" && req && res && isRender(req)) { + if (state.inflight >= state.max) { + state.shed += 1; + res.statusCode = 503; + res.setHeader("Retry-After", retryAfter); + res.setHeader("Cache-Control", "no-store"); + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.end("Server busy, retry shortly"); + return true; + } + state.inflight += 1; + let released = false; + let graceTimer = null; + const release = () => { + if (released) return; + released = true; + if (graceTimer) clearTimeout(graceTimer); + state.inflight -= 1; + }; + // finish = the response was written: the render is over. When the + // client has already gone, a destroyed response never emits finish, so + // the end() call itself (Next writing its last byte) releases too. + res.once("finish", release); + const originalEnd = res.end; + res.end = function end() { + release(); + return originalEnd.apply(this, arguments); + }; + // close before the render ends = the client went away while it runs on; + // keep the slot for the grace period, or until end(), whichever first. + res.once("close", () => { + if (released) return; + state.abandoned += 1; + if (abandonedGraceMs === 0) { + release(); + return; + } + graceTimer = setTimeout(release, abandonedGraceMs); + graceTimer.unref(); + }); + } + return originalEmit.apply(this, arguments); + }; + + // One line per minute at most, and only when something was shed, so a + // saturated process shows up in the container log without flooding it. + let lastShed = 0; + setInterval(() => { + if (state.shed === lastShed) return; + process.stderr.write( + `[ssr-admission] shed ${state.shed - lastShed} requests in the last 60s (inflight=${state.inflight}, max=${state.max}, abandoned=${state.abandoned})\n` + ); + lastShed = state.shed; + }, 60_000).unref(); + + process.stderr.write(`[ssr-admission] max in-flight renders per process: ${max}\n`); +} else if (rawMax !== undefined && rawMax !== "" && !(Number.isFinite(max) && max > 0) && max !== 0) { + process.stderr.write(`[ssr-admission] SSR_MAX_INFLIGHT=${JSON.stringify(rawMax)} is not a positive number; disabled\n`); +} else { + process.stderr.write("[ssr-admission] disabled (SSR_MAX_INFLIGHT unset or 0)\n"); +}