From d11729a0cc1a1efc089e14d43ffde577db6d4f3c Mon Sep 17 00:00:00 2001 From: feruzm Date: Wed, 12 Aug 2026 21:20:11 +0000 Subject: [PATCH 1/3] hosting: page the SEO feed at the bridge's own limit The bridge asserts limit into [1:20] and answers an error rather than a shorter list, so the single limit=100 call failed every tenant's SEO pass in production and no robots.txt, sitemap.xml or rss.xml was ever written. Walk pages of 20 with an exclusive cursor up to the same wanted depth, under one budget for the whole walk as well as the per-call timeout, and terminate on a short page, a repeated page or an unusable record. --- .../api/src/services/seo-files.test.ts | 56 +++++++++ .../hosting/api/src/services/seo-files.ts | 108 ++++++++++++------ 2 files changed, 126 insertions(+), 38 deletions(-) diff --git a/apps/self-hosted/hosting/api/src/services/seo-files.test.ts b/apps/self-hosted/hosting/api/src/services/seo-files.test.ts index 4cd827bf19..2fe9702a7e 100644 --- a/apps/self-hosted/hosting/api/src/services/seo-files.test.ts +++ b/apps/self-hosted/hosting/api/src/services/seo-files.test.ts @@ -122,6 +122,11 @@ describe('fetchTenantPosts', () => { undefined, expect.any(AbortSignal), ); + // The bridge asserts limit into [1:20] and ERRORS above it, so a page may + // never ask for more; asking for 100 failed every tenant in production. + for (const call of mocks.callRPC.mock.calls) { + expect(call[1].limit).toBeLessThanOrEqual(20); + } const community = { ...TENANT, @@ -141,6 +146,57 @@ describe('fetchTenantPosts', () => { ); }); + it('walks pages with an exclusive cursor up to the wanted depth', async () => { + // Six full pages exist; the walk wants 100 posts, so it stops after five. + const page = (n: number) => + Array.from({ length: 20 }, (_, i) => ({ + author: 'alice', + permlink: `p${n}-${i}`, + created: '2026-08-01T00:00:00', + })); + mocks.callRPC.mockImplementation(async (_m: string, params: any) => + page(Number(params.start_permlink?.split('-')[0]?.replace('p', '') ?? -1) + 1), + ); + + const posts = await fetchTenantPosts(TENANT); + expect(posts).toHaveLength(100); + expect(mocks.callRPC).toHaveBeenCalledTimes(5); + // Each page after the first carries the previous page's last post as the + // cursor, and every returned post is distinct. + expect(mocks.callRPC.mock.calls[1][1]).toMatchObject({ + start_author: 'alice', + start_permlink: 'p0-19', + }); + expect(new Set(posts.map((p) => p.permlink)).size).toBe(100); + }); + + it('stops on a short page instead of asking for one more', async () => { + mocks.callRPC.mockResolvedValue( + Array.from({ length: 3 }, (_, i) => ({ + author: 'alice', + permlink: `only-${i}`, + created: '2026-08-01T00:00:00', + })), + ); + const posts = await fetchTenantPosts(TENANT); + expect(posts).toHaveLength(3); + expect(mocks.callRPC).toHaveBeenCalledTimes(1); + }); + + it('terminates when a node echoes the cursor post back forever', async () => { + // An inclusive-cursor node would otherwise repeat its last page for ever: + // a full page whose posts are all already seen ends the walk. + const repeated = Array.from({ length: 20 }, (_, i) => ({ + author: 'alice', + permlink: `same-${i}`, + created: '2026-08-01T00:00:00', + })); + mocks.callRPC.mockResolvedValue(repeated); + const posts = await fetchTenantPosts(TENANT); + expect(posts).toHaveLength(20); + expect(mocks.callRPC).toHaveBeenCalledTimes(2); + }); + it('throws on a malformed response so stale files are kept, never blanked', async () => { mocks.callRPC.mockResolvedValue({ nope: true }); await expect(fetchTenantPosts(TENANT)).rejects.toThrow('malformed'); diff --git a/apps/self-hosted/hosting/api/src/services/seo-files.ts b/apps/self-hosted/hosting/api/src/services/seo-files.ts index b3acf1939e..4f5d26dd17 100644 --- a/apps/self-hosted/hosting/api/src/services/seo-files.ts +++ b/apps/self-hosted/hosting/api/src/services/seo-files.ts @@ -43,8 +43,19 @@ export function canonicalPostUrl( return `https://ecency.com/@${author}/${permlink}`; } -/** How many posts feeds and sitemaps carry; one bridge page. */ +/** + * How many posts feeds and sitemaps carry, and how they are collected. + * + * The bridge asserts `limit` into [1:20] and answers an ERROR, not a shorter + * list, when asked for more: a single limit=100 call failed every tenant's + * pass in production, so the wanted depth is PAGED at the bridge's own page + * size. The whole walk carries one deadline as well as the per-call one, so + * a chain that answers slowly costs a bounded pass rather than page count + * times the per-call timeout. + */ const POST_LIMIT = 100; +const BRIDGE_PAGE_LIMIT = 20; +const FETCH_BUDGET_MS = 30_000; /** A pass regenerates a tenant's files only when they are older than this. */ export const SEO_FRESH_MS = 30 * 60 * 1000; /** The background pass is patient but never unbounded. */ @@ -92,45 +103,66 @@ async function boundedCall(method: string, params: object): Promise { /** The tenant's latest posts, the same feeds the archive itself pages. */ export async function fetchTenantPosts(tenant: Tenant): Promise { const { community, communityId } = isCommunityTenant(tenant); - const raw = community - ? await boundedCall('bridge.get_ranked_posts', { - sort: 'created', - tag: communityId, - limit: POST_LIMIT, - observer: '', - }) - : await boundedCall('bridge.get_account_posts', { - sort: 'posts', - account: tenant.username, - limit: POST_LIMIT, - observer: '', - }); - // A malformed answer is an ERROR, never an empty blog: returning [] here - // would overwrite a good sitemap and feed with empty ones and mark them - // fresh, while throwing lets the sync pass keep yesterday's files. - if (!Array.isArray(raw)) { - throw new Error('malformed bridge feed response'); - } - // Every field the builders touch is type-checked here: a malformed record - // (a numeric date, a missing permlink) is dropped or normalized instead of - // failing the tenant's whole SEO pass on an .endsWith of a number. + const method = community + ? 'bridge.get_ranked_posts' + : 'bridge.get_account_posts'; + const feedParams = community + ? { sort: 'created', tag: communityId, observer: '' } + : { sort: 'posts', account: tenant.username, observer: '' }; + const posts: TenantPost[] = []; - for (const p of raw as any[]) { - if ( - typeof p?.author !== 'string' || - typeof p?.permlink !== 'string' || - typeof p?.created !== 'string' - ) { - continue; - } - posts.push({ - author: p.author, - permlink: p.permlink, - title: typeof p.title === 'string' ? p.title : '', - created: p.created, - updated: typeof p.updated === 'string' ? p.updated : undefined, - body: typeof p.body === 'string' ? p.body : undefined, + const seen = new Set(); + const deadline = Date.now() + FETCH_BUDGET_MS; + let cursor: { start_author: string; start_permlink: string } | null = null; + + while (posts.length < POST_LIMIT) { + const raw = await boundedCall(method, { + ...feedParams, + limit: Math.min(BRIDGE_PAGE_LIMIT, POST_LIMIT - posts.length), + ...(cursor ?? {}), }); + // A malformed answer is an ERROR, never an empty blog: returning [] here + // would overwrite a good sitemap and feed with empty ones and mark them + // fresh, while throwing lets the sync pass keep yesterday's files. + if (!Array.isArray(raw)) { + throw new Error('malformed bridge feed response'); + } + const page = raw as any[]; + // Every field the builders touch is type-checked here: a malformed record + // (a numeric date, a missing permlink) is dropped or normalized instead of + // failing the tenant's whole SEO pass on an .endsWith of a number. + let added = 0; + let last: { author: string; permlink: string } | null = null; + for (const p of page) { + if ( + typeof p?.author !== 'string' || + typeof p?.permlink !== 'string' || + typeof p?.created !== 'string' + ) { + continue; + } + last = { author: p.author, permlink: p.permlink }; + // The cursor is exclusive on today's bridge, but a node that echoes the + // start post back would otherwise repeat a page forever; the identity + // set makes the walk terminate either way. + const key = `${p.author}/${p.permlink}`; + if (seen.has(key)) continue; + seen.add(key); + added++; + posts.push({ + author: p.author, + permlink: p.permlink, + title: typeof p.title === 'string' ? p.title : '', + created: p.created, + updated: typeof p.updated === 'string' ? p.updated : undefined, + body: typeof p.body === 'string' ? p.body : undefined, + }); + } + // A short page is the end of the feed; no new posts or no usable record + // means paging further cannot help. Either way, stop. + if (page.length < BRIDGE_PAGE_LIMIT || added === 0 || !last) break; + if (Date.now() >= deadline) break; + cursor = { start_author: last.author, start_permlink: last.permlink }; } return posts; } From e0f97eeeafee7d5d7f34d250b2e0431c900507b9 Mon Sep 17 00:00:00 2001 From: feruzm Date: Wed, 12 Aug 2026 21:25:36 +0000 Subject: [PATCH 2/3] hosting: reserve a cursor slot and bound each page by the walk budget --- .../api/src/services/seo-files.test.ts | 56 +++++++++++++++++++ .../hosting/api/src/services/seo-files.ts | 44 ++++++++++----- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/apps/self-hosted/hosting/api/src/services/seo-files.test.ts b/apps/self-hosted/hosting/api/src/services/seo-files.test.ts index 2fe9702a7e..706c5dd360 100644 --- a/apps/self-hosted/hosting/api/src/services/seo-files.test.ts +++ b/apps/self-hosted/hosting/api/src/services/seo-files.test.ts @@ -170,6 +170,62 @@ describe('fetchTenantPosts', () => { expect(new Set(posts.map((p) => p.permlink)).size).toBe(100); }); + it('reaches full depth on a node whose cursor is inclusive', async () => { + // Such a node echoes the cursor post back as the first entry. Without a + // reserved slot the final short ask spends one on the duplicate and the + // walk ends one post shy of the wanted depth. + let n = 0; + mocks.callRPC.mockImplementation(async (_m: string, params: any) => { + const page: any[] = []; + if (params.start_permlink) { + page.push({ + author: 'alice', + permlink: params.start_permlink, + created: '2026-08-01T00:00:00', + }); + } + while (page.length < params.limit) { + page.push({ author: 'alice', permlink: `q${n++}`, created: '2026-08-01T00:00:00' }); + } + return page; + }); + + const posts = await fetchTenantPosts(TENANT); + expect(posts).toHaveLength(100); + expect(new Set(posts.map((p) => p.permlink)).size).toBe(100); + // Follow-up pages ask for one more than they need, never above the cap. + for (const call of mocks.callRPC.mock.calls) { + expect(call[1].limit).toBeLessThanOrEqual(20); + } + }); + + it('never starts a page that cannot finish inside the walk budget', async () => { + // Each page eats most of the budget; the walk must stop rather than let + // a further per-call timeout run past the deadline. + let elapsed = 0; + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => elapsed); + try { + mocks.callRPC.mockImplementation(async (_m: string, params: any) => { + elapsed += 9_000; + return Array.from({ length: params.limit }, (_, i) => ({ + author: 'alice', + permlink: `r${elapsed}-${i}`, + created: '2026-08-01T00:00:00', + })); + }); + + await fetchTenantPosts(TENANT); + // 30s budget, 9s per page: four pages fit, the fifth is not started. + expect(mocks.callRPC).toHaveBeenCalledTimes(4); + // The last call is bounded by what is LEFT of the budget, not the + // full per-call timeout. + const lastTimeout = mocks.callRPC.mock.calls.at(-1)![2]; + expect(lastTimeout).toBeLessThanOrEqual(30_000 - 27_000); + } finally { + nowSpy.mockRestore(); + } + }); + it('stops on a short page instead of asking for one more', async () => { mocks.callRPC.mockResolvedValue( Array.from({ length: 3 }, (_, i) => ({ diff --git a/apps/self-hosted/hosting/api/src/services/seo-files.ts b/apps/self-hosted/hosting/api/src/services/seo-files.ts index 4f5d26dd17..7491eb7cbb 100644 --- a/apps/self-hosted/hosting/api/src/services/seo-files.ts +++ b/apps/self-hosted/hosting/api/src/services/seo-files.ts @@ -56,6 +56,8 @@ export function canonicalPostUrl( const POST_LIMIT = 100; const BRIDGE_PAGE_LIMIT = 20; const FETCH_BUDGET_MS = 30_000; +/** Below this much budget left, a further page is not worth starting. */ +const MIN_CALL_MS = 1_000; /** A pass regenerates a tenant's files only when they are older than this. */ export const SEO_FRESH_MS = 30 * 60 * 1000; /** The background pass is patient but never unbounded. */ @@ -90,11 +92,15 @@ function isCommunityTenant(tenant: Tenant): { * and the controller cancels the request outright at the same deadline, so a * timed-out fetch stops consuming a socket instead of racing on unobserved. */ -async function boundedCall(method: string, params: object): Promise { +async function boundedCall( + method: string, + params: object, + timeoutMs: number = RPC_TIMEOUT_MS, +): Promise { const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), RPC_TIMEOUT_MS); + const timer = setTimeout(() => controller.abort(), timeoutMs); try { - return await callRPC(method, params, RPC_TIMEOUT_MS, undefined, controller.signal); + return await callRPC(method, params, timeoutMs, undefined, controller.signal); } finally { clearTimeout(timer); } @@ -116,11 +122,23 @@ export async function fetchTenantPosts(tenant: Tenant): Promise { let cursor: { start_author: string; start_permlink: string } | null = null; while (posts.length < POST_LIMIT) { - const raw = await boundedCall(method, { - ...feedParams, - limit: Math.min(BRIDGE_PAGE_LIMIT, POST_LIMIT - posts.length), - ...(cursor ?? {}), - }); + // The whole-walk budget bounds the NEXT call rather than being noticed + // after it: checking only afterwards let a page that starts just inside + // the budget run a further full per-call timeout past it. + const remaining = deadline - Date.now(); + if (remaining <= MIN_CALL_MS) break; + // One extra slot on follow-up pages. Cursor inclusivity varies by node + // (today's answer both feeds exclusively, this repo's own paginator + // documents the opposite), and on an inclusive node the echoed cursor + // would eat a slot from the last short ask and end the walk one post + // early. The identity set below still does the actual de-duplication. + const need = POST_LIMIT - posts.length; + const ask = Math.min(BRIDGE_PAGE_LIMIT, cursor ? need + 1 : need); + const raw = await boundedCall( + method, + { ...feedParams, limit: ask, ...(cursor ?? {}) }, + Math.min(RPC_TIMEOUT_MS, remaining), + ); // A malformed answer is an ERROR, never an empty blog: returning [] here // would overwrite a good sitemap and feed with empty ones and mark them // fresh, while throwing lets the sync pass keep yesterday's files. @@ -158,13 +176,13 @@ export async function fetchTenantPosts(tenant: Tenant): Promise { body: typeof p.body === 'string' ? p.body : undefined, }); } - // A short page is the end of the feed; no new posts or no usable record - // means paging further cannot help. Either way, stop. - if (page.length < BRIDGE_PAGE_LIMIT || added === 0 || !last) break; - if (Date.now() >= deadline) break; + // A page shorter than what it was ASKED for is the end of the feed; no + // new posts or no usable record means paging further cannot help. + if (page.length < ask || added === 0 || !last) break; cursor = { start_author: last.author, start_permlink: last.permlink }; } - return posts; + // The reserved slot can overshoot by one on an exclusive node. + return posts.slice(0, POST_LIMIT); } export function buildRobotsTxt(tenant: Tenant): string { From 5fb63d3c5e42cfff393ff9db2f4a1654bcffc4f2 Mon Sep 17 00:00:00 2001 From: feruzm Date: Wed, 12 Aug 2026 21:27:49 +0000 Subject: [PATCH 3/3] hosting: narrow feed records without any casts --- .../api/src/services/seo-files.test.ts | 22 ++++++-- .../hosting/api/src/services/seo-files.ts | 56 +++++++++++-------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/apps/self-hosted/hosting/api/src/services/seo-files.test.ts b/apps/self-hosted/hosting/api/src/services/seo-files.test.ts index 706c5dd360..0323f203fc 100644 --- a/apps/self-hosted/hosting/api/src/services/seo-files.test.ts +++ b/apps/self-hosted/hosting/api/src/services/seo-files.test.ts @@ -40,6 +40,20 @@ const CUSTOM_TENANT = { customDomainVerified: true, } as any; +/** The feed params the walk sends, as the mocks below read them. */ +interface FeedParams { + limit: number; + start_author?: string; + start_permlink?: string; +} + +/** One record as the bridge answers it, shaped for these mocks. */ +interface FeedRecord { + author: string; + permlink: string; + created: string; +} + const POSTS = [ { author: 'alice', @@ -154,7 +168,7 @@ describe('fetchTenantPosts', () => { permlink: `p${n}-${i}`, created: '2026-08-01T00:00:00', })); - mocks.callRPC.mockImplementation(async (_m: string, params: any) => + mocks.callRPC.mockImplementation(async (_m: string, params: FeedParams) => page(Number(params.start_permlink?.split('-')[0]?.replace('p', '') ?? -1) + 1), ); @@ -175,8 +189,8 @@ describe('fetchTenantPosts', () => { // reserved slot the final short ask spends one on the duplicate and the // walk ends one post shy of the wanted depth. let n = 0; - mocks.callRPC.mockImplementation(async (_m: string, params: any) => { - const page: any[] = []; + mocks.callRPC.mockImplementation(async (_m: string, params: FeedParams) => { + const page: FeedRecord[] = []; if (params.start_permlink) { page.push({ author: 'alice', @@ -205,7 +219,7 @@ describe('fetchTenantPosts', () => { let elapsed = 0; const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => elapsed); try { - mocks.callRPC.mockImplementation(async (_m: string, params: any) => { + mocks.callRPC.mockImplementation(async (_m: string, params: FeedParams) => { elapsed += 9_000; return Array.from({ length: params.limit }, (_, i) => ({ author: 'alice', diff --git a/apps/self-hosted/hosting/api/src/services/seo-files.ts b/apps/self-hosted/hosting/api/src/services/seo-files.ts index 7491eb7cbb..c80947e25f 100644 --- a/apps/self-hosted/hosting/api/src/services/seo-files.ts +++ b/apps/self-hosted/hosting/api/src/services/seo-files.ts @@ -72,6 +72,33 @@ interface TenantPost { body?: string; } +/** + * One feed record, narrowed from an unknown chain answer. Every field the + * builders touch is checked here: a malformed record (a numeric date, a + * missing permlink) is dropped or normalized instead of failing the + * tenant's whole SEO pass on an .endsWith of a number. + */ +function toTenantPost(entry: unknown): TenantPost | null { + if (typeof entry !== 'object' || entry === null) return null; + const record = entry as Record; + const { author, permlink, created, title, updated, body } = record; + if ( + typeof author !== 'string' || + typeof permlink !== 'string' || + typeof created !== 'string' + ) { + return null; + } + return { + author, + permlink, + created, + title: typeof title === 'string' ? title : '', + updated: typeof updated === 'string' ? updated : undefined, + body: typeof body === 'string' ? body : undefined, + }; +} + function isCommunityTenant(tenant: Tenant): { community: boolean; communityId: string; @@ -145,36 +172,21 @@ export async function fetchTenantPosts(tenant: Tenant): Promise { if (!Array.isArray(raw)) { throw new Error('malformed bridge feed response'); } - const page = raw as any[]; - // Every field the builders touch is type-checked here: a malformed record - // (a numeric date, a missing permlink) is dropped or normalized instead of - // failing the tenant's whole SEO pass on an .endsWith of a number. + const page: unknown[] = raw; let added = 0; let last: { author: string; permlink: string } | null = null; - for (const p of page) { - if ( - typeof p?.author !== 'string' || - typeof p?.permlink !== 'string' || - typeof p?.created !== 'string' - ) { - continue; - } - last = { author: p.author, permlink: p.permlink }; + for (const entry of page) { + const post = toTenantPost(entry); + if (!post) continue; + last = { author: post.author, permlink: post.permlink }; // The cursor is exclusive on today's bridge, but a node that echoes the // start post back would otherwise repeat a page forever; the identity // set makes the walk terminate either way. - const key = `${p.author}/${p.permlink}`; + const key = `${post.author}/${post.permlink}`; if (seen.has(key)) continue; seen.add(key); added++; - posts.push({ - author: p.author, - permlink: p.permlink, - title: typeof p.title === 'string' ? p.title : '', - created: p.created, - updated: typeof p.updated === 'string' ? p.updated : undefined, - body: typeof p.body === 'string' ? p.body : undefined, - }); + posts.push(post); } // A page shorter than what it was ASKED for is the end of the feed; no // new posts or no usable record means paging further cannot help.