From e4edfa749bb7108f093b2f391bc694b1ca5265cb Mon Sep 17 00:00:00 2001 From: jakeaturner Date: Mon, 5 Oct 2026 19:40:15 +0000 Subject: [PATCH] feat: support fetching images in private books --- create-localstack-resources.sh | 14 +++++ src/lib/cxOneAssetFetcher.ts | 108 +++++++++++++++++++++++++++++++++ src/services/epub.ts | 12 ++-- src/services/imageProcessor.ts | 73 +++++++++++++--------- src/services/library.ts | 16 +++++ 5 files changed, 186 insertions(+), 37 deletions(-) create mode 100644 src/lib/cxOneAssetFetcher.ts diff --git a/create-localstack-resources.sh b/create-localstack-resources.sh index 212e776..1848e87 100755 --- a/create-localstack-resources.sh +++ b/create-localstack-resources.sh @@ -105,6 +105,20 @@ awslocal ssm put-parameter \ --overwrite echo "Created SSM parameter: /libkeys/production/chem/secret" +awslocal ssm put-parameter \ + --name "/libkeys/production/eng/key" \ + --type "SecureString" \ + --value "${LIBKEYS_PROD_ENG_KEY}" \ + --overwrite +echo "Created SSM parameter: /libkeys/production/eng/key" + +awslocal ssm put-parameter \ + --name "/libkeys/production/eng/secret" \ + --type "SecureString" \ + --value "${LIBKEYS_PROD_ENG_SECRET}" \ + --overwrite +echo "Created SSM parameter: /libkeys/production/eng/secret" + awslocal ssm put-parameter \ --name "/libkeys/production/math/key" \ --type "SecureString" \ diff --git a/src/lib/cxOneAssetFetcher.ts b/src/lib/cxOneAssetFetcher.ts new file mode 100644 index 0000000..676f5bc --- /dev/null +++ b/src/lib/cxOneAssetFetcher.ts @@ -0,0 +1,108 @@ +import axios, { AxiosError } from 'axios'; +import { libraryNameKeysWDev } from '../librariesmap'; +import { LibraryService } from '../services/library'; +import { USER_AGENT } from '../util/util'; +import { CXOneRateLimiter } from './cxOneRateLimiter'; + +const DEFAULT_TIMEOUT_MS = 30_000; +const LIBRARY_HOST = /^([a-z0-9-]+)\.libretexts\.org$/i; +const RATE_LIMITED_HOST = /(^|\.)libretexts\.org$/i; +const MAX_REDIRECTS = 5; + +export interface FetchedAsset { + /** True when the anonymous request was refused and the asset needed a server token. */ + authenticated: boolean; + contentType: string | undefined; + data: Buffer; +} + +/** + * One initialized client per library, shared across jobs. Init reads the token pair from + * SSM, so it's done lazily and only for libraries that actually serve a restricted asset. + */ +const libraryClients = new Map>(); + +function getLibraryClient(lib: string): Promise { + let client = libraryClients.get(lib); + if (!client) { + client = (async () => { + const service = new LibraryService({ lib }); + await service.init(); + return service; + })(); + // Drop failed inits so a transient SSM error doesn't poison the library for the worker's lifetime. + client.catch(() => libraryClients.delete(lib)); + libraryClients.set(lib, client); + } + return client; +} + +/** Returns the library key for a known LibreTexts library host, or null for anything else. */ +function getLibraryFromURL(url: string): string | null { + const match = LIBRARY_HOST.exec(new URL(url).hostname); + const lib = match?.[1]?.toLowerCase(); + return lib && libraryNameKeysWDev.includes(lib) ? lib : null; +} + +/** CXOne answers an anonymous request for a restricted file with a redirect to its login page. */ +function isLoginRedirect(location: URL): boolean { + return location.pathname.startsWith('/@app/auth/'); +} + +class AuthRequiredError extends Error {} + +/** + * Downloads an asset referenced by book content (typically an image under `/@api/deki/files`). + * + * The request is anonymous first, which is what public content needs and keeps CDN caching + * intact. Files attached to pages with a Private (or otherwise restricted) CXOne permission + * don't fail outright: they 302 to the login page, which axios would happily follow and return + * as a 200 HTML document. Redirects are therefore followed by hand. A login redirect (or a + * 401/403) from a known library host triggers one retry with the same server token used to + * fetch the page content. The token is only ever sent to the original library host, never to + * a redirect target such as a signed storage URL. + */ +export async function fetchCXOneAsset(url: string, opts: { timeout?: number } = {}): Promise { + const lib = getLibraryFromURL(url); + const origin = new URL(url).origin; + + const download = async (headers: Record): Promise> => { + let target = new URL(url); + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + // CXOne-hosted files count against the same API budget as page content; CDN assets don't. + if (RATE_LIMITED_HOST.test(target.hostname)) await CXOneRateLimiter.waitUntilAPIAvailable(); + const response = await axios.get(target.toString(), { + headers: { 'User-Agent': USER_AGENT, ...(target.origin === origin && headers) }, + maxRedirects: 0, + responseType: 'arraybuffer', + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + validateStatus: (status) => status >= 200 && status < 400, + }); + if (response.status >= 300) { + const location = response.headers['location']; + if (!location) throw new Error(`Redirect without location for ${target}`); + const next = new URL(location, target); + if (isLoginRedirect(next)) throw new AuthRequiredError(`Login required for ${url}`); + target = next; + continue; + } + const contentType = response.headers['content-type'] as string | undefined; + // An HTML document is never a valid asset; it's an error or login page that slipped through. + if (contentType?.startsWith('text/html')) + throw new AuthRequiredError(`Received HTML instead of asset for ${url}`); + return { contentType, data: Buffer.from(response.data as unknown as ArrayBuffer) }; + } + throw new Error(`Too many redirects for ${url}`); + }; + + try { + return { authenticated: false, ...(await download({})) }; + } catch (error) { + const status = error instanceof AxiosError ? error.response?.status : undefined; + const authRequired = error instanceof AuthRequiredError || status === 401 || status === 403; + if (!lib || !authRequired) throw error; + } + + const client = await getLibraryClient(lib); + return { authenticated: true, ...(await download(client.getAuthHeaders())) }; +} diff --git a/src/services/epub.ts b/src/services/epub.ts index 24d2ab4..3bac634 100644 --- a/src/services/epub.ts +++ b/src/services/epub.ts @@ -10,13 +10,14 @@ import { v4 as uuid } from 'uuid'; import { XMLBuilder } from 'fast-xml-parser'; import Archiver from 'archiver'; import * as cheerio from 'cheerio'; -import { runBatchedPromises, USER_AGENT } from '../util/util'; -import axios, { AxiosError } from 'axios'; +import { runBatchedPromises } from '../util/util'; +import { AxiosError } from 'axios'; import mime from 'mime'; import beautify from 'js-beautify'; import { decodePresentationalEntities } from '../util/htmlFilters'; import PageID from '../util/pageID'; import { optimizeImageBuffer } from '../util/imageOptimizer'; +import { fetchCXOneAsset } from '../lib/cxOneAssetFetcher'; import { PassThrough } from 'node:stream'; import { Upload } from '@aws-sdk/lib-storage'; import { nullProgressReporter, type ProgressReporter } from '../lib/jobProgress'; @@ -323,12 +324,7 @@ export class EPUBService { } // - const imageResp = await axios.get(fqImageURL, { - responseType: 'arraybuffer', - headers: { 'User-Agent': USER_AGENT }, - }); - const originalData = Buffer.from(imageResp.data, 'binary'); - const responseMimeType = imageResp.headers['content-type'] as string | undefined; + const { contentType: responseMimeType, data: originalData } = await fetchCXOneAsset(fqImageURL); // CMS images are routinely far larger than any reader displays them at; embedding // them verbatim is what makes EPUBs (like PDFs) balloon. `null` means the image is diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index ef59b6e..3270af0 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -1,16 +1,15 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import { isAbsolute, join, relative, resolve } from 'node:path'; -import axios from 'axios'; import * as cheerio from 'cheerio'; import { LogLayer } from 'loglayer'; import pLimit from 'p-limit'; -import { CXOneRateLimiter } from '../lib/cxOneRateLimiter'; +import mime from 'mime'; +import { fetchCXOneAsset } from '../lib/cxOneAssetFetcher'; import { Environment } from '../lib/environment'; import { log as logService } from '../lib/log'; import { BookPageInfo } from '../types/book'; import { optimizeImageBuffer } from '../util/imageOptimizer'; -import { USER_AGENT } from '../util/util'; import { nullProgressReporter, type ProgressReporter } from '../lib/jobProgress'; const DEFAULT_FETCH_CONCURRENCY = 8; @@ -29,11 +28,13 @@ const DEFAULT_JPEG_QUALITY = 80; /** * The only shape a generated image file name may take: a SHA-1 hex digest plus one of the - * two extensions {@link optimizeImageBuffer} can emit. Everything written to (or read back + * two extensions {@link optimizeImageBuffer} can emit, or the extension of an original kept + * verbatim (see {@link ImageProcessor.resolveLocalPath}). Everything written to (or read back * from) the image directory is checked against this, so a hostile `src` in book content can * never steer a filesystem operation at a path we didn't construct ourselves. */ -const OPTIMIZED_FILE_NAME = /^[0-9a-f]{40}\.(?:jpg|png)$/; +const OPTIMIZED_FILE_NAME = /^[0-9a-f]{40}\.(?:jpg|jpeg|png|gif|webp|svg)$/; +const ORIGINAL_EXTENSIONS = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg']); export interface ImageProcessingSummary { /** Total bytes of the source images that were replaced. */ @@ -177,10 +178,7 @@ export class ImageProcessor { : trimmed.startsWith('//') ? `https:${trimmed}` : `https://${subdomain}.libretexts.org/${trimmed.replace(/^\//, '')}`; - const url = new URL(raw); - // SVG is vector — re-encoding it would only make it bigger and blurrier. - if (url.pathname.toLowerCase().endsWith('.svg')) return null; - return url.toString(); + return new URL(raw).toString(); } catch { return null; } @@ -233,24 +231,24 @@ export class ImageProcessor { if (cached !== undefined) return cached; try { - // CXOne-hosted files count against the same API budget as page content; CDN assets don't. - if (/(^|\.)libretexts\.org$/i.test(new URL(url).hostname)) { - await CXOneRateLimiter.waitUntilAPIAvailable(); - } - - const response = await axios.get(url, { - headers: { 'User-Agent': USER_AGENT }, - responseType: 'arraybuffer', + const { + authenticated, + contentType, + data: original, + } = await fetchCXOneAsset(url, { timeout: IMAGE_FETCH_TIMEOUT_MS, }); - const original = Buffer.from(response.data as unknown as ArrayBuffer); - const optimized = await optimizeImageBuffer(original, { - jpegQuality: this._jpegQuality, - maxHeight: this._maxHeight, - maxWidth: this._maxWidth, - }); - if (!optimized) { + // SVG is vector — re-encoding it would only make it bigger and blurrier. + const isSVG = contentType?.includes('svg') || new URL(url).pathname.toLowerCase().endsWith('.svg'); + const optimized = isSVG + ? null + : await optimizeImageBuffer(original, { + jpegQuality: this._jpegQuality, + maxHeight: this._maxHeight, + maxWidth: this._maxWidth, + }); + if (!optimized && !authenticated) { // Vector, animated, or already smaller than anything we'd produce. Leaving the // remote URL in place lets Prince fetch it as before. this._summary.skipped++; @@ -258,13 +256,23 @@ export class ImageProcessor { return null; } - const fileName = `${createHash('sha1').update(url).digest('hex')}.${optimized.extension}`; + // An image that needed a server token (e.g. on a Private page) is unreachable for + // Prince, so when it can't be optimized the original bytes are kept locally instead. + const extension = optimized?.extension ?? this.originalExtension(contentType, isSVG); + if (!extension) throw new Error(`Unsupported content type for restricted image: ${contentType}`); + const data = optimized?.data ?? original; + + const fileName = `${createHash('sha1').update(url).digest('hex')}.${extension}`; const filePath = this.resolveOutputPath(fileName); - await fs.writeFile(filePath, optimized.data); + await fs.writeFile(filePath, data); - this._summary.optimized++; - this._summary.bytesBefore += original.length; - this._summary.bytesAfter += optimized.data.length; + if (optimized) { + this._summary.optimized++; + this._summary.bytesBefore += original.length; + this._summary.bytesAfter += data.length; + } else { + this._summary.skipped++; + } this._cache.set(url, filePath); return filePath; } catch (error) { @@ -277,6 +285,13 @@ export class ImageProcessor { } } + /** File extension for an original image kept verbatim, limited to formats Prince renders. */ + private originalExtension(contentType: string | undefined, isSVG: boolean): string | null { + if (isSVG) return 'svg'; + const extension = contentType ? mime.getExtension(contentType.split(';')[0].trim()) : null; + return extension && ORIGINAL_EXTENSIONS.has(extension) ? extension : null; + } + /** * Maps a generated file name to its absolute path inside the image directory. * diff --git a/src/services/library.ts b/src/services/library.ts index fd12524..d01aca7 100644 --- a/src/services/library.ts +++ b/src/services/library.ts @@ -78,6 +78,22 @@ export class LibraryService { return this.expertClient!; } + /** + * Mints a fresh server token header for requests made outside the Expert client, such as + * raw file downloads. Server tokens embed a timestamp, so callers should request a new + * header per request rather than holding on to one. + */ + public getAuthHeaders(): Record { + this._ensureInitialized(); + const token = this.expertClient!.auth.ServerToken({ + key: this.keyPair!.key, + secret: this.keyPair!.secret, + user: this.user, + }).getToken(); + if (!token) throw new Error(`[${this.logName}] Failed to mint server token for ${this.lib}`); + return { 'X-Deki-Token': token }; + } + private _generateLibrariesSSMClient(): LibrariesSSMClient | null { try { const libTokenPairPath = process.env.AWS_SSM_LIB_TOKEN_PAIR_PATH || '/libkeys/production';