take a screenshot with scrcpy, suggestion need

[saknarak]

I don't know how to decode/encdoe from h264 to jpeg
without write to mp4 first and then call ffmpeg to get frame snapshot

here is my code

const scrcpy = client.scrcpy({
  crop: '640:480:0:300',
})
scrcpy.on('config', config => {
  console.log('config=', config)
})
scrcpy.on('frame', async data => {
  // how to decode data into jpeg, following code from test task
  let dm = await beamcoder.demuxer() // HELP NEED
  let dec = beamcoder.decoder({ demuxer: dm, stream_index: 0 }) // Create a decoder
  let packet = await dm.read()
  let decResult = await dec.decode(packet)
  let enc = beamcoder.encoder({ // Create an encoder for JPEG data
    name: 'mjpeg', // FFmpeg does not have an encoder called 'jpeg'
    width: dec.width,
    height: dec.height,
    pix_fmt: dec.pix_fmt.indexOf('422') >= 0 ? 'yuvj422p' : 'yuvj420p',
    time_base: [1, 1],
  })
  let jpegResult = await enc.encode(decResult.frames[0]) // Encode the frame
  await enc.flush() // Tidy the encoder
  await fs.promises.writeFile(`./out/snap_${new Date().getTime()}.jpeg`, jpegResult.packets[0].data)
})

let info = await scrcpy.start()
// firstFrame is undefined, it would be nice if firstFrame is a frame data
let frame = await info.firstFrame

PS. I want to make screenshot from some restrict app that not allow to.
Only scrcpy tools can cast stream out , but it can't take a screenshot.

[UrielCh]

I had fork beamcoder for that purpose, but I did not complet the developpement, it's only work with low screen resolution.

[Garfonso]

I am not sure if this is still relevant for the OP, but a web search still leads here.

My solution is to spawn FFmpeg as a child process and feed it the data from scrcpy via stdin and get decoded frames via stdout. I then use this data for image processing in OpenCV. It is the raw bitmap... you probably can save that as a bitmap or feed it into some PNG compressor or so to get a screenshot.

Example code would look something like this:

import { spawn } from 'child_process';

function setupScrcpy {
        const wmOutput = await adb.execOut('wm size', 'utf8');
        const match = wmOutput.match(/(\d+)x(\d+)/);
        if (!match) {
            throw new Error(`Cannot parse screen size from: ${wmOutput}`);
        }
        const screensize = { x: Number(match[1]), y: Number(match[2]) };
        console.log('Screen size:', screensize);
        const config = {
            maxFps: 25,
            maxSize: 600,
            showTouches: false,
            stayAwake: true,
            control: true,
            audio: false,
            sendFrameMeta: true,
            clipboardAutosync: false,
        };
        let width = 1080;
        let height = 2400;
        let imageSize = width * height * 3;
        const ffmpeg = spawn(
            'ffmpeg',
            [
                '-use_wallclock_as_timestamps',
                '1', // Use wall-clock for timestamps
                '-f',
                'h264', // Input-Format ist H.264
                //'-s',
                //`${272}x${608}`, // input resolution
                '-i',
                'pipe:0', // Eingabe: Piped Stream
                //'-r',
                //config.maxFps.toString(), // Framerate
                '-fflags',
                'nobuffer', // no buffer
                '-flags',
                'low_delay', // low delay
                '-probesize',
                '32', // reduce probesize for faster start
                '-analyzeduration',
                '0', // disable duration analysis
                '-vsync',
                'drop', //drop frames if too fast?
                //'-preset',
                //'ultrafast', //try to be very fast -> nah, is for output.. idiot gpt...
                '-f',
                'rawvideo', // Ausgabe: Roh-Frame-Daten
                '-pix_fmt',
                'rgb24', // Pixel-Format für OpenCV
                //'-vf', 'scale=640:480', // Optional: Skalieren auf gewünschte Größe
                //'-fps', '5',          // Optional: Framerate auf 5 FPS begrenzen
                'pipe:1', // Ausgabe zu stdout (Rohdaten direkt an den Node.js-Prozess)
            ],
            {
                stdio: ['pipe', 'pipe', 'pipe'],
            },
        );
        let currentImageBuffer = Buffer.allocUnsafe(0);
        ffmpeg.stdout.on('data', (data: Buffer) => {
            currentImageBuffer = Buffer.concat([currentImageBuffer, data]);
            let frame;
            while (currentImageBuffer.length >= imageSize) {
                frame = currentImageBuffer.subarray(0, imageSize);
                currentImageBuffer = currentImageBuffer.subarray(imageSize);
            }
            //store only the last frame.
            if (frame) {
                console.log('got a complete frame');
            }
        });
        ffmpeg.stdin.on('error', (err: Error) => {
            console.error('FFmpeg stdin error:', err);
        });
        ffmpeg.stdin.on('close', () => console.log('FFmpeg stdin closed'));
        ffmpeg.stderr.on('data', (data: Buffer) => {
            console.error(`FFmpeg Error: ${data.toString()}`);
        });
        ffmpeg.on('close', (code: number) => {
            console.log(`FFmpeg process exited with code ${code}`);
        });
        ffmpeg.on('spawn', () => console.log('FFmpeg spawned'));

        const scrcpy = new Scrcpy(device.adb, config);
        scrcpy.on('frame', frame => {
            ffmpeg.stdin.write(frame.data);
        });
        scrcpy.on('config', config => {
            ffmpeg.stdin.write(config.data);
            console.log('Config:', config);
            width = config.croppedWidth;
            height = config.croppedHeight;
            imageSize = width * height * 3;
            currentImageBuffer = Buffer.allocUnsafe(imageSize);
        });
        scrcpy.on('error', error => console.error('Scrcpy error:', error, error.stack));
        scrcpy.on('disconnect', () => {
            console.log('Scrcpy disconnected');
            ffmpeg.stdin.end();
        });
        scrcpy.on('raw', raw => {
            ffmpeg.stdin.write(raw);
        });
        await scrcpy.start();
}