Page the per-tenant SEO feed at the bridge's own limit

apps/self-hosted/hosting/api/src/services/seo-files.test.ts

@@ -40,6 +40,20 @@ const CUSTOM_TENANT = {
   customDomainVerified: true,
 } as any;
 
+/** The feed params the walk sends, as the mocks below read them. */
+interface FeedParams {
+  limit: number;
+  start_author?: string;
+  start_permlink?: string;
+}
+
+/** One record as the bridge answers it, shaped for these mocks. */
+interface FeedRecord {
+  author: string;
+  permlink: string;
+  created: string;
+}
+
 const POSTS = [
   {
     author: 'alice',
@@ -122,6 +136,11 @@ describe('fetchTenantPosts', () => {
       undefined,
       expect.any(AbortSignal),
     );
+    // The bridge asserts limit into [1:20] and ERRORS above it, so a page may
+    // never ask for more; asking for 100 failed every tenant in production.
+    for (const call of mocks.callRPC.mock.calls) {
+      expect(call[1].limit).toBeLessThanOrEqual(20);
+    }
 
     const community = {
       ...TENANT,
@@ -141,6 +160,113 @@ describe('fetchTenantPosts', () => {
     );
   });
 
+  it('walks pages with an exclusive cursor up to the wanted depth', async () => {
+    // Six full pages exist; the walk wants 100 posts, so it stops after five.
+    const page = (n: number) =>
+      Array.from({ length: 20 }, (_, i) => ({
+        author: 'alice',
+        permlink: `p${n}-${i}`,
+        created: '2026-08-01T00:00:00',
+      }));
+    mocks.callRPC.mockImplementation(async (_m: string, params: FeedParams) =>
+      page(Number(params.start_permlink?.split('-')[0]?.replace('p', '') ?? -1) + 1),
+    );
+
+    const posts = await fetchTenantPosts(TENANT);
+    expect(posts).toHaveLength(100);
+    expect(mocks.callRPC).toHaveBeenCalledTimes(5);
+    // Each page after the first carries the previous page's last post as the
+    // cursor, and every returned post is distinct.
+    expect(mocks.callRPC.mock.calls[1][1]).toMatchObject({
+      start_author: 'alice',
+      start_permlink: 'p0-19',
+    });
+    expect(new Set(posts.map((p) => p.permlink)).size).toBe(100);
+  });
+
+  it('reaches full depth on a node whose cursor is inclusive', async () => {
+    // Such a node echoes the cursor post back as the first entry. Without a
+    // reserved slot the final short ask spends one on the duplicate and the
+    // walk ends one post shy of the wanted depth.
+    let n = 0;
+    mocks.callRPC.mockImplementation(async (_m: string, params: FeedParams) => {
+      const page: FeedRecord[] = [];
+      if (params.start_permlink) {
+        page.push({
+          author: 'alice',
+          permlink: params.start_permlink,
+          created: '2026-08-01T00:00:00',
+        });
+      }
+      while (page.length < params.limit) {
+        page.push({ author: 'alice', permlink: `q${n++}`, created: '2026-08-01T00:00:00' });
+      }
+      return page;
+    });
+
+    const posts = await fetchTenantPosts(TENANT);
+    expect(posts).toHaveLength(100);
+    expect(new Set(posts.map((p) => p.permlink)).size).toBe(100);
+    // Follow-up pages ask for one more than they need, never above the cap.
+    for (const call of mocks.callRPC.mock.calls) {
+      expect(call[1].limit).toBeLessThanOrEqual(20);
+    }
+  });
+
+  it('never starts a page that cannot finish inside the walk budget', async () => {
+    // Each page eats most of the budget; the walk must stop rather than let
+    // a further per-call timeout run past the deadline.
+    let elapsed = 0;
+    const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => elapsed);
+    try {
+      mocks.callRPC.mockImplementation(async (_m: string, params: FeedParams) => {
+        elapsed += 9_000;
+        return Array.from({ length: params.limit }, (_, i) => ({
+          author: 'alice',
+          permlink: `r${elapsed}-${i}`,
+          created: '2026-08-01T00:00:00',
+        }));
+      });
+
+      await fetchTenantPosts(TENANT);
+      // 30s budget, 9s per page: four pages fit, the fifth is not started.
+      expect(mocks.callRPC).toHaveBeenCalledTimes(4);
+      // The last call is bounded by what is LEFT of the budget, not the
+      // full per-call timeout.
+      const lastTimeout = mocks.callRPC.mock.calls.at(-1)![2];
+      expect(lastTimeout).toBeLessThanOrEqual(30_000 - 27_000);
+    } finally {
+      nowSpy.mockRestore();
+    }
+  });
+
+  it('stops on a short page instead of asking for one more', async () => {
+    mocks.callRPC.mockResolvedValue(
+      Array.from({ length: 3 }, (_, i) => ({
+        author: 'alice',
+        permlink: `only-${i}`,
+        created: '2026-08-01T00:00:00',
+      })),
+    );
+    const posts = await fetchTenantPosts(TENANT);
+    expect(posts).toHaveLength(3);
+    expect(mocks.callRPC).toHaveBeenCalledTimes(1);
+  });
+
+  it('terminates when a node echoes the cursor post back forever', async () => {
+    // An inclusive-cursor node would otherwise repeat its last page for ever:
+    // a full page whose posts are all already seen ends the walk.
+    const repeated = Array.from({ length: 20 }, (_, i) => ({
+      author: 'alice',
+      permlink: `same-${i}`,
+      created: '2026-08-01T00:00:00',
+    }));
+    mocks.callRPC.mockResolvedValue(repeated);
+    const posts = await fetchTenantPosts(TENANT);
+    expect(posts).toHaveLength(20);
+    expect(mocks.callRPC).toHaveBeenCalledTimes(2);
+  });
+
   it('throws on a malformed response so stale files are kept, never blanked', async () => {
     mocks.callRPC.mockResolvedValue({ nope: true });
     await expect(fetchTenantPosts(TENANT)).rejects.toThrow('malformed');

apps/self-hosted/hosting/api/src/services/seo-files.ts

@@ -43,8 +43,21 @@ export function canonicalPostUrl(
   return `https://ecency.com/@${author}/${permlink}`;
 }
 
-/** How many posts feeds and sitemaps carry; one bridge page. */
+/**
+ * How many posts feeds and sitemaps carry, and how they are collected.
+ *
+ * The bridge asserts `limit` into [1:20] and answers an ERROR, not a shorter
+ * list, when asked for more: a single limit=100 call failed every tenant's
+ * pass in production, so the wanted depth is PAGED at the bridge's own page
+ * size. The whole walk carries one deadline as well as the per-call one, so
+ * a chain that answers slowly costs a bounded pass rather than page count
+ * times the per-call timeout.
+ */
 const POST_LIMIT = 100;
+const BRIDGE_PAGE_LIMIT = 20;
+const FETCH_BUDGET_MS = 30_000;
+/** Below this much budget left, a further page is not worth starting. */
+const MIN_CALL_MS = 1_000;
 /** A pass regenerates a tenant's files only when they are older than this. */
 export const SEO_FRESH_MS = 30 * 60 * 1000;
 /** The background pass is patient but never unbounded. */
@@ -59,6 +72,33 @@ interface TenantPost {
   body?: string;
 }
 
+/**
+ * One feed record, narrowed from an unknown chain answer. Every field the
+ * builders touch is checked here: a malformed record (a numeric date, a
+ * missing permlink) is dropped or normalized instead of failing the
+ * tenant's whole SEO pass on an .endsWith of a number.
+ */
+function toTenantPost(entry: unknown): TenantPost | null {
+  if (typeof entry !== 'object' || entry === null) return null;
+  const record = entry as Record<string, unknown>;
+  const { author, permlink, created, title, updated, body } = record;
+  if (
+    typeof author !== 'string' ||
+    typeof permlink !== 'string' ||
+    typeof created !== 'string'
+  ) {
+    return null;
+  }
+  return {
+    author,
+    permlink,
+    created,
+    title: typeof title === 'string' ? title : '',
+    updated: typeof updated === 'string' ? updated : undefined,
+    body: typeof body === 'string' ? body : undefined,
+  };
+}
+
 function isCommunityTenant(tenant: Tenant): {
   community: boolean;
   communityId: string;
@@ -79,11 +119,15 @@ function isCommunityTenant(tenant: Tenant): {
  * and the controller cancels the request outright at the same deadline, so a
  * timed-out fetch stops consuming a socket instead of racing on unobserved.
  */
-async function boundedCall<T>(method: string, params: object): Promise<T> {
+async function boundedCall<T>(
+  method: string,
+  params: object,
+  timeoutMs: number = RPC_TIMEOUT_MS,
+): Promise<T> {
   const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), RPC_TIMEOUT_MS);
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
   try {
-    return await callRPC<T>(method, params, RPC_TIMEOUT_MS, undefined, controller.signal);
+    return await callRPC<T>(method, params, timeoutMs, undefined, controller.signal);
   } finally {
     clearTimeout(timer);
   }
@@ -92,47 +136,65 @@ async function boundedCall<T>(method: string, params: object): Promise<T> {
 /** The tenant's latest posts, the same feeds the archive itself pages. */
 export async function fetchTenantPosts(tenant: Tenant): Promise<TenantPost[]> {
   const { community, communityId } = isCommunityTenant(tenant);
-  const raw = community
-    ? await boundedCall<unknown>('bridge.get_ranked_posts', {
-        sort: 'created',
-        tag: communityId,
-        limit: POST_LIMIT,
-        observer: '',
-      })
-    : await boundedCall<unknown>('bridge.get_account_posts', {
-        sort: 'posts',
-        account: tenant.username,
-        limit: POST_LIMIT,
-        observer: '',
-      });
-  // A malformed answer is an ERROR, never an empty blog: returning [] here
-  // would overwrite a good sitemap and feed with empty ones and mark them
-  // fresh, while throwing lets the sync pass keep yesterday's files.
-  if (!Array.isArray(raw)) {
-    throw new Error('malformed bridge feed response');
-  }
-  // Every field the builders touch is type-checked here: a malformed record
-  // (a numeric date, a missing permlink) is dropped or normalized instead of
-  // failing the tenant's whole SEO pass on an .endsWith of a number.
+  const method = community
+    ? 'bridge.get_ranked_posts'
+    : 'bridge.get_account_posts';
+  const feedParams = community
+    ? { sort: 'created', tag: communityId, observer: '' }
+    : { sort: 'posts', account: tenant.username, observer: '' };
+
   const posts: TenantPost[] = [];
-  for (const p of raw as any[]) {
-    if (
-      typeof p?.author !== 'string' ||
-      typeof p?.permlink !== 'string' ||
-      typeof p?.created !== 'string'
-    ) {
-      continue;
+  const seen = new Set<string>();
+  const deadline = Date.now() + FETCH_BUDGET_MS;
+  let cursor: { start_author: string; start_permlink: string } | null = null;
+
+  while (posts.length < POST_LIMIT) {
+    // The whole-walk budget bounds the NEXT call rather than being noticed
+    // after it: checking only afterwards let a page that starts just inside
+    // the budget run a further full per-call timeout past it.
+    const remaining = deadline - Date.now();
+    if (remaining <= MIN_CALL_MS) break;
+    // One extra slot on follow-up pages. Cursor inclusivity varies by node
+    // (today's answer both feeds exclusively, this repo's own paginator
+    // documents the opposite), and on an inclusive node the echoed cursor
+    // would eat a slot from the last short ask and end the walk one post
+    // early. The identity set below still does the actual de-duplication.
+    const need = POST_LIMIT - posts.length;
+    const ask = Math.min(BRIDGE_PAGE_LIMIT, cursor ? need + 1 : need);
+    const raw = await boundedCall<unknown>(
+      method,
+      { ...feedParams, limit: ask, ...(cursor ?? {}) },
+      Math.min(RPC_TIMEOUT_MS, remaining),
+    );
+    // A malformed answer is an ERROR, never an empty blog: returning [] here
+    // would overwrite a good sitemap and feed with empty ones and mark them
+    // fresh, while throwing lets the sync pass keep yesterday's files.
+    if (!Array.isArray(raw)) {
+      throw new Error('malformed bridge feed response');
+    }
+    const page: unknown[] = raw;
+    let added = 0;
+    let last: { author: string; permlink: string } | null = null;
+    for (const entry of page) {
+      const post = toTenantPost(entry);
+      if (!post) continue;
+      last = { author: post.author, permlink: post.permlink };
+      // The cursor is exclusive on today's bridge, but a node that echoes the
+      // start post back would otherwise repeat a page forever; the identity
+      // set makes the walk terminate either way.
+      const key = `${post.author}/${post.permlink}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      added++;
+      posts.push(post);
     }
-    posts.push({
-      author: p.author,
-      permlink: p.permlink,
-      title: typeof p.title === 'string' ? p.title : '',
-      created: p.created,
-      updated: typeof p.updated === 'string' ? p.updated : undefined,
-      body: typeof p.body === 'string' ? p.body : undefined,
-    });
+    // A page shorter than what it was ASKED for is the end of the feed; no
+    // new posts or no usable record means paging further cannot help.
+    if (page.length < ask || added === 0 || !last) break;
+    cursor = { start_author: last.author, start_permlink: last.permlink };
   }
-  return posts;
+  // The reserved slot can overshoot by one on an exclusive node.
+  return posts.slice(0, POST_LIMIT);
 }
 
 export function buildRobotsTxt(tenant: Tenant): string {