From c6f316b4fed3536d2276e5e7e84e9d9578f4ac4b Mon Sep 17 00:00:00 2001 From: Perry Zhu Date: Sat, 1 Nov 2025 13:53:56 -0700 Subject: [PATCH] fix: lambda script --- scripts/Dockerfile.script | 8 ++-- scripts/build-lambda.sh | 14 ++++--- scripts/src/lambda-handler.ts | 8 +++- scripts/src/main.ts | 79 ++++++++++++++++++++++++++++++++--- 4 files changed, 94 insertions(+), 15 deletions(-) diff --git a/scripts/Dockerfile.script b/scripts/Dockerfile.script index 6410d0f..18b1590 100644 --- a/scripts/Dockerfile.script +++ b/scripts/Dockerfile.script @@ -15,10 +15,12 @@ RUN apt-get update && \ make \ cmake \ unzip \ - libcurl4-openssl-dev + libcurl4-openssl-dev && \ + rm -rf /var/lib/apt/lists/* RUN npm install aws-lambda-ric RUN npm ci --only=production -ENV PLAYWRIGHT_BROWSERS_PATH=0 -RUN npx playwright install chromium --with-deps +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright +RUN npx playwright install chromium --with-deps && \ + npx playwright install-deps chromium ENTRYPOINT ["/usr/local/bin/npx", "aws-lambda-ric"] CMD ["dist/lambda-handler.handler"] \ No newline at end of file diff --git a/scripts/build-lambda.sh b/scripts/build-lambda.sh index 59fcc47..0145364 100755 --- a/scripts/build-lambda.sh +++ b/scripts/build-lambda.sh @@ -1,7 +1,11 @@ #!/bin/bash set -e -docker buildx build --platform linux/arm64 --provenance=false -f Dockerfile.script -t lambda-script:test . -docker run --platform linux/arm64 -v ~/.aws-lambda-rie:/aws-lambda -p 9000:8080 \ - --entrypoint /aws-lambda/aws-lambda-rie \ - lambda-script:test \ - /usr/local/bin/npx aws-lambda-ric dist/lambda-handler.handler \ No newline at end of file +docker buildx build --platform linux/amd64 --provenance=false -f Dockerfile.script -t pwp/pexels-image-scraper-lambda . +# docker run --platform linux/amd64 -v ~/.aws-lambda-rie:/aws-lambda -p 9000:8080 \ +# --entrypoint /aws-lambda/aws-lambda-rie \ +# lambda-script:test \ +# /usr/local/bin/npx aws-lambda-ric dist/lambda-handler.handler + +docker tag pwp/pexels-image-scraper-lambda:latest 279892746640.dkr.ecr.us-west-2.amazonaws.com/pwp/pexels-image-scraper-lambda:latest + +docker push 279892746640.dkr.ecr.us-west-2.amazonaws.com/pwp/pexels-image-scraper-lambda:latest \ No newline at end of file diff --git a/scripts/src/lambda-handler.ts b/scripts/src/lambda-handler.ts index 1dab506..eed8ab9 100644 --- a/scripts/src/lambda-handler.ts +++ b/scripts/src/lambda-handler.ts @@ -14,9 +14,11 @@ export async function handler( // eslint-disable-next-line @typescript-eslint/no-unused-vars _context: Context, ): Promise { + console.log("Lambda handler started"); try { const pexelsUrl = "https://www.pexels.com/@perry-z-1662054943/featured-uploads/"; + console.log(`Fetching images from: ${pexelsUrl}`); const imageLinks = await getImageLinksPlaywright(pexelsUrl); if (imageLinks && imageLinks.length > 0) { @@ -42,7 +44,11 @@ export async function handler( }; } } catch (error) { - console.log(`Error in lambdaHandler: ${error}`); + console.error(`Error in lambdaHandler: ${error}`); + if (error instanceof Error) { + console.error(`Error message: ${error.message}`); + console.error(`Error stack: ${error.stack}`); + } return { statusCode: 500, body: JSON.stringify({ diff --git a/scripts/src/main.ts b/scripts/src/main.ts index 3a89451..b0fe812 100644 --- a/scripts/src/main.ts +++ b/scripts/src/main.ts @@ -51,18 +51,68 @@ async function findLoadMoreButton( async function getImageLinksPlaywright(url: string): Promise { let browser = null; + let context = null; + let page = null; + + // Set environment variable to use system Chromium in Lambda + process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || '/ms-playwright/chromium-*/chrome-linux/chrome'; + try { + console.log("Environment:", { + PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH, + PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, + NODE_ENV: process.env.NODE_ENV, + }); + + console.log("Launching browser..."); + const launchStart = Date.now(); browser = await chromium.launch({ headless: true, - args: ["--disable-gpu", "--disable-blink-features=AutomationControlled"], + timeout: 120000, + args: [ + "--disable-gpu", + "--disable-blink-features=AutomationControlled", + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-dev-shm-usage", + "--disable-software-rasterizer", + "--disable-extensions", + "--disable-background-networking", + "--disable-sync", + "--metrics-recording-only", + "--mute-audio", + "--no-first-run", + "--safebrowsing-disable-auto-update", + "--disable-component-update", + "--disable-default-apps", + "--no-zygote", + "--disable-accelerated-2d-canvas", + "--disable-webgl", + ], }); + const launchDuration = Date.now() - launchStart; + console.log(`Browser launched successfully in ${launchDuration}ms`); + + if (!browser.isConnected()) { + throw new Error("Browser disconnected after launch"); + } - const context = await browser.newContext({ + console.log("Creating browser context..."); + context = await browser.newContext({ userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", }); - - const page = await context.newPage(); + console.log("Browser context created successfully"); + + console.log("Creating new page..."); + try { + page = await context.newPage(); + console.log("New page created successfully"); + } catch (pageError) { + console.error("Failed to create new page"); + console.error(`Browser connected: ${browser.isConnected()}`); + throw pageError; + } console.log("Fetching URL with Playwright..."); await page.goto(url, { @@ -128,10 +178,27 @@ async function getImageLinksPlaywright(url: string): Promise { return processedImgLinks; } catch (error) { console.error(`An error occurred: ${error}`); + if (error instanceof Error) { + console.error(`Error stack: ${error.stack}`); + } return []; } finally { - if (browser) { - await browser.close(); + console.log("Cleaning up resources..."); + try { + if (page) { + await page.close(); + console.log("Page closed"); + } + if (context) { + await context.close(); + console.log("Context closed"); + } + if (browser) { + await browser.close(); + console.log("Browser closed"); + } + } catch (cleanupError) { + console.error(`Error during cleanup: ${cleanupError}`); } } }