feat: support fetching images in private books

create-localstack-resources.sh

@@ -105,6 +105,20 @@ awslocal ssm put-parameter \
     --overwrite
 echo "Created SSM parameter: /libkeys/production/chem/secret"
 
+awslocal ssm put-parameter \
+    --name "/libkeys/production/eng/key" \
+    --type "SecureString" \
+    --value "${LIBKEYS_PROD_ENG_KEY}" \
+    --overwrite
+echo "Created SSM parameter: /libkeys/production/eng/key"
+
+awslocal ssm put-parameter \
+    --name "/libkeys/production/eng/secret" \
+    --type "SecureString" \
+    --value "${LIBKEYS_PROD_ENG_SECRET}" \
+    --overwrite
+echo "Created SSM parameter: /libkeys/production/eng/secret"
+
 awslocal ssm put-parameter \
     --name "/libkeys/production/math/key" \
     --type "SecureString" \

src/lib/cxOneAssetFetcher.ts

@@ -0,0 +1,108 @@
+import axios, { AxiosError } from 'axios';
+import { libraryNameKeysWDev } from '../librariesmap';
+import { LibraryService } from '../services/library';
+import { USER_AGENT } from '../util/util';
+import { CXOneRateLimiter } from './cxOneRateLimiter';
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const LIBRARY_HOST = /^([a-z0-9-]+)\.libretexts\.org$/i;
+const RATE_LIMITED_HOST = /(^|\.)libretexts\.org$/i;
+const MAX_REDIRECTS = 5;
+
+export interface FetchedAsset {
+  /** True when the anonymous request was refused and the asset needed a server token. */
+  authenticated: boolean;
+  contentType: string | undefined;
+  data: Buffer;
+}
+
+/**
+ * One initialized client per library, shared across jobs. Init reads the token pair from
+ * SSM, so it's done lazily and only for libraries that actually serve a restricted asset.
+ */
+const libraryClients = new Map<string, Promise<LibraryService>>();
+
+function getLibraryClient(lib: string): Promise<LibraryService> {
+  let client = libraryClients.get(lib);
+  if (!client) {
+    client = (async () => {
+      const service = new LibraryService({ lib });
+      await service.init();
+      return service;
+    })();
+    // Drop failed inits so a transient SSM error doesn't poison the library for the worker's lifetime.
+    client.catch(() => libraryClients.delete(lib));
+    libraryClients.set(lib, client);
+  }
+  return client;
+}
+
+/** Returns the library key for a known LibreTexts library host, or null for anything else. */
+function getLibraryFromURL(url: string): string | null {
+  const match = LIBRARY_HOST.exec(new URL(url).hostname);
+  const lib = match?.[1]?.toLowerCase();
+  return lib && libraryNameKeysWDev.includes(lib) ? lib : null;
+}
+
+/** CXOne answers an anonymous request for a restricted file with a redirect to its login page. */
+function isLoginRedirect(location: URL): boolean {
+  return location.pathname.startsWith('/@app/auth/');
+}
+
+class AuthRequiredError extends Error {}
+
+/**
+ * Downloads an asset referenced by book content (typically an image under `/@api/deki/files`).
+ *
+ * The request is anonymous first, which is what public content needs and keeps CDN caching
+ * intact. Files attached to pages with a Private (or otherwise restricted) CXOne permission
+ * don't fail outright: they 302 to the login page, which axios would happily follow and return
+ * as a 200 HTML document. Redirects are therefore followed by hand. A login redirect (or a
+ * 401/403) from a known library host triggers one retry with the same server token used to
+ * fetch the page content. The token is only ever sent to the original library host, never to
+ * a redirect target such as a signed storage URL.
+ */
+export async function fetchCXOneAsset(url: string, opts: { timeout?: number } = {}): Promise<FetchedAsset> {
+  const lib = getLibraryFromURL(url);
+  const origin = new URL(url).origin;
+
+  const download = async (headers: Record<string, string>): Promise<Omit<FetchedAsset, 'authenticated'>> => {
+    let target = new URL(url);
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+      // CXOne-hosted files count against the same API budget as page content; CDN assets don't.
+      if (RATE_LIMITED_HOST.test(target.hostname)) await CXOneRateLimiter.waitUntilAPIAvailable();
+      const response = await axios.get<ArrayBuffer>(target.toString(), {
+        headers: { 'User-Agent': USER_AGENT, ...(target.origin === origin && headers) },
+        maxRedirects: 0,
+        responseType: 'arraybuffer',
+        timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS,
+        validateStatus: (status) => status >= 200 && status < 400,
+      });
+      if (response.status >= 300) {
+        const location = response.headers['location'];
+        if (!location) throw new Error(`Redirect without location for ${target}`);
+        const next = new URL(location, target);
+        if (isLoginRedirect(next)) throw new AuthRequiredError(`Login required for ${url}`);
+        target = next;
+        continue;
+      }
+      const contentType = response.headers['content-type'] as string | undefined;
+      // An HTML document is never a valid asset; it's an error or login page that slipped through.
+      if (contentType?.startsWith('text/html'))
+        throw new AuthRequiredError(`Received HTML instead of asset for ${url}`);
+      return { contentType, data: Buffer.from(response.data as unknown as ArrayBuffer) };
+    }
+    throw new Error(`Too many redirects for ${url}`);
+  };
+
+  try {
+    return { authenticated: false, ...(await download({})) };
+  } catch (error) {
+    const status = error instanceof AxiosError ? error.response?.status : undefined;
+    const authRequired = error instanceof AuthRequiredError || status === 401 || status === 403;
+    if (!lib || !authRequired) throw error;
+  }
+
+  const client = await getLibraryClient(lib);
+  return { authenticated: true, ...(await download(client.getAuthHeaders())) };
+}

src/services/epub.ts

@@ -10,13 +10,14 @@ import { v4 as uuid } from 'uuid';
 import { XMLBuilder } from 'fast-xml-parser';
 import Archiver from 'archiver';
 import * as cheerio from 'cheerio';
-import { runBatchedPromises, USER_AGENT } from '../util/util';
-import axios, { AxiosError } from 'axios';
+import { runBatchedPromises } from '../util/util';
+import { AxiosError } from 'axios';
 import mime from 'mime';
 import beautify from 'js-beautify';
 import { decodePresentationalEntities } from '../util/htmlFilters';
 import PageID from '../util/pageID';
 import { optimizeImageBuffer } from '../util/imageOptimizer';
+import { fetchCXOneAsset } from '../lib/cxOneAssetFetcher';
 import { PassThrough } from 'node:stream';
 import { Upload } from '@aws-sdk/lib-storage';
 import { nullProgressReporter, type ProgressReporter } from '../lib/jobProgress';
@@ -323,12 +324,7 @@ export class EPUBService {
           }
 
           // <download image, store metadata, and replace URL>
-          const imageResp = await axios.get(fqImageURL, {
-            responseType: 'arraybuffer',
-            headers: { 'User-Agent': USER_AGENT },
-          });
-          const originalData = Buffer.from(imageResp.data, 'binary');
-          const responseMimeType = imageResp.headers['content-type'] as string | undefined;
+          const { contentType: responseMimeType, data: originalData } = await fetchCXOneAsset(fqImageURL);
 
           // CMS images are routinely far larger than any reader displays them at; embedding
           // them verbatim is what makes EPUBs (like PDFs) balloon. `null` means the image is

src/services/imageProcessor.ts

@@ -1,16 +1,15 @@
 import { createHash } from 'node:crypto';
 import fs from 'node:fs/promises';
 import { isAbsolute, join, relative, resolve } from 'node:path';
-import axios from 'axios';
 import * as cheerio from 'cheerio';
 import { LogLayer } from 'loglayer';
 import pLimit from 'p-limit';
-import { CXOneRateLimiter } from '../lib/cxOneRateLimiter';
+import mime from 'mime';
+import { fetchCXOneAsset } from '../lib/cxOneAssetFetcher';
 import { Environment } from '../lib/environment';
 import { log as logService } from '../lib/log';
 import { BookPageInfo } from '../types/book';
 import { optimizeImageBuffer } from '../util/imageOptimizer';
-import { USER_AGENT } from '../util/util';
 import { nullProgressReporter, type ProgressReporter } from '../lib/jobProgress';
 
 const DEFAULT_FETCH_CONCURRENCY = 8;
@@ -29,11 +28,13 @@ const DEFAULT_JPEG_QUALITY = 80;
 
 /**
  * The only shape a generated image file name may take: a SHA-1 hex digest plus one of the
- * two extensions {@link optimizeImageBuffer} can emit. Everything written to (or read back
+ * two extensions {@link optimizeImageBuffer} can emit, or the extension of an original kept
+ * verbatim (see {@link ImageProcessor.resolveLocalPath}). Everything written to (or read back
  * from) the image directory is checked against this, so a hostile `src` in book content can
  * never steer a filesystem operation at a path we didn't construct ourselves.
  */
-const OPTIMIZED_FILE_NAME = /^[0-9a-f]{40}\.(?:jpg|png)$/;
+const OPTIMIZED_FILE_NAME = /^[0-9a-f]{40}\.(?:jpg|jpeg|png|gif|webp|svg)$/;
+const ORIGINAL_EXTENSIONS = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg']);
 
 export interface ImageProcessingSummary {
   /** Total bytes of the source images that were replaced. */
@@ -177,10 +178,7 @@ export class ImageProcessor {
         : trimmed.startsWith('//')
           ? `https:${trimmed}`
           : `https://${subdomain}.libretexts.org/${trimmed.replace(/^\//, '')}`;
-      const url = new URL(raw);
-      // SVG is vector — re-encoding it would only make it bigger and blurrier.
-      if (url.pathname.toLowerCase().endsWith('.svg')) return null;
-      return url.toString();
+      return new URL(raw).toString();
     } catch {
       return null;
     }
@@ -233,38 +231,48 @@ export class ImageProcessor {
     if (cached !== undefined) return cached;
 
     try {
-      // CXOne-hosted files count against the same API budget as page content; CDN assets don't.
-      if (/(^|\.)libretexts\.org$/i.test(new URL(url).hostname)) {
-        await CXOneRateLimiter.waitUntilAPIAvailable();
-      }
-
-      const response = await axios.get<ArrayBuffer>(url, {
-        headers: { 'User-Agent': USER_AGENT },
-        responseType: 'arraybuffer',
+      const {
+        authenticated,
+        contentType,
+        data: original,
+      } = await fetchCXOneAsset(url, {
         timeout: IMAGE_FETCH_TIMEOUT_MS,
       });
-      const original = Buffer.from(response.data as unknown as ArrayBuffer);
 
-      const optimized = await optimizeImageBuffer(original, {
-        jpegQuality: this._jpegQuality,
-        maxHeight: this._maxHeight,
-        maxWidth: this._maxWidth,
-      });
-      if (!optimized) {
+      // SVG is vector — re-encoding it would only make it bigger and blurrier.
+      const isSVG = contentType?.includes('svg') || new URL(url).pathname.toLowerCase().endsWith('.svg');
+      const optimized = isSVG
+        ? null
+        : await optimizeImageBuffer(original, {
+            jpegQuality: this._jpegQuality,
+            maxHeight: this._maxHeight,
+            maxWidth: this._maxWidth,
+          });
+      if (!optimized && !authenticated) {
         // Vector, animated, or already smaller than anything we'd produce. Leaving the
         // remote URL in place lets Prince fetch it as before.
         this._summary.skipped++;
         this._cache.set(url, null);
         return null;
       }
 
-      const fileName = `${createHash('sha1').update(url).digest('hex')}.${optimized.extension}`;
+      // An image that needed a server token (e.g. on a Private page) is unreachable for
+      // Prince, so when it can't be optimized the original bytes are kept locally instead.
+      const extension = optimized?.extension ?? this.originalExtension(contentType, isSVG);
+      if (!extension) throw new Error(`Unsupported content type for restricted image: ${contentType}`);
+      const data = optimized?.data ?? original;
+
+      const fileName = `${createHash('sha1').update(url).digest('hex')}.${extension}`;
       const filePath = this.resolveOutputPath(fileName);
-      await fs.writeFile(filePath, optimized.data);
+      await fs.writeFile(filePath, data);
 
-      this._summary.optimized++;
-      this._summary.bytesBefore += original.length;
-      this._summary.bytesAfter += optimized.data.length;
+      if (optimized) {
+        this._summary.optimized++;
+        this._summary.bytesBefore += original.length;
+        this._summary.bytesAfter += data.length;
+      } else {
+        this._summary.skipped++;
+      }
       this._cache.set(url, filePath);
       return filePath;
     } catch (error) {
@@ -277,6 +285,13 @@ export class ImageProcessor {
     }
   }
 
+  /** File extension for an original image kept verbatim, limited to formats Prince renders. */
+  private originalExtension(contentType: string | undefined, isSVG: boolean): string | null {
+    if (isSVG) return 'svg';
+    const extension = contentType ? mime.getExtension(contentType.split(';')[0].trim()) : null;
+    return extension && ORIGINAL_EXTENSIONS.has(extension) ? extension : null;
+  }
+
   /**
    * Maps a generated file name to its absolute path inside the image directory.
    *

src/services/library.ts

@@ -78,6 +78,22 @@ export class LibraryService {
     return this.expertClient!;
   }
 
+  /**
+   * Mints a fresh server token header for requests made outside the Expert client, such as
+   * raw file downloads. Server tokens embed a timestamp, so callers should request a new
+   * header per request rather than holding on to one.
+   */
+  public getAuthHeaders(): Record<string, string> {
+    this._ensureInitialized();
+    const token = this.expertClient!.auth.ServerToken({
+      key: this.keyPair!.key,
+      secret: this.keyPair!.secret,
+      user: this.user,
+    }).getToken();
+    if (!token) throw new Error(`[${this.logName}] Failed to mint server token for ${this.lib}`);
+    return { 'X-Deki-Token': token };
+  }
+
   private _generateLibrariesSSMClient(): LibrariesSSMClient | null {
     try {
       const libTokenPairPath = process.env.AWS_SSM_LIB_TOKEN_PAIR_PATH || '/libkeys/production';