Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/cli/src/commands/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ function probeVideo(filePath: string): VideoMeta | undefined {
if (!ffprobePath) return undefined;
const raw = execFileSync(
ffprobePath,
["-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", filePath],
["-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "--", filePath],
{ encoding: "utf-8", timeout: 15_000 },
);

Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/utils/webmAlphaCheck.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ function probeWebmAlpha(filePath: string): WebmAlphaProbe {
"stream=codec_name:stream_tags=alpha_mode",
"-of",
"json",
"--",
filePath,
],
{ encoding: "utf-8", timeout: 15_000 },
Expand Down
3 changes: 2 additions & 1 deletion packages/cli/src/whisper/transcribe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ function getMediaDurationSeconds(filePath: string): number | null {
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
"--",
filePath,
],
{ encoding: "utf-8", timeout: 10_000 },
Expand Down Expand Up @@ -329,7 +330,7 @@ function isWav16kMono(filePath: string): boolean {
if (!ffprobePath) return false;
const raw = execFileSync(
ffprobePath,
["-v", "quiet", "-print_format", "json", "-show_streams", filePath],
["-v", "quiet", "-print_format", "json", "-show_streams", "--", filePath],
{ encoding: "utf-8", timeout: 10_000 },
);
const parsed: {
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ export {
quantizeTimeToFrame,
type MediaVisualStyleProperty,
} from "./inline-scripts/parityContract";
export { redactTelemetryString } from "./telemetryRedaction";
export { redactKnownPaths, redactTelemetryString } from "./telemetryRedaction";
export { isSafePath, resolveWithinProject } from "./safePath";
export type {
HyperframePickerApi,
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/mediaGradeAnalyzer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ function probeMedia(mediaPath: string, ffprobePath: string): GradeMediaProbe {
"stream=color_space,color_transfer,color_primaries,pix_fmt,duration:format=duration",
"-of",
"json",
"--",
mediaPath,
],
{ encoding: "utf8", timeout: 5_000, stdio: ["ignore", "pipe", "pipe"] },
Expand Down
139 changes: 138 additions & 1 deletion packages/core/src/telemetryRedaction.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { redactTelemetryString } from "./telemetryRedaction.js";
import { redactKnownPaths, redactTelemetryString } from "./telemetryRedaction.js";

describe("redactTelemetryString", () => {
it("redacts macOS, Linux, Windows, file URLs, and URL query strings", () => {
Expand All @@ -16,4 +16,141 @@ describe("redactTelemetryString", () => {
),
).toBe("[path] [path] [path] [path] [file-url] https://example.com/video.mp4?…");
});

// The redactor used to enumerate roots (/Users, /home, /opt, /tmp, …). Any
// root outside that list reached telemetry verbatim, which is most of them.
it.each([
"/data/media/interview.mov",
"/mnt2/nfs/share/take3.wav",
"/srv2/renders/2026/final.mp4",
"/nix/store/abc123/asset.png",
])("redacts the non-allowlisted absolute root in %s", (path) => {
const out = redactTelemetryString(`ffprobe failed reading ${path}`);
expect(out).not.toContain("/");
expect(out).toContain("[path]");
});

it("redacts relative paths, including a dash-prefixed one", () => {
expect(redactTelemetryString("could not open ./assets/-weird-name.mp3")).toBe(
"could not open [path]",
);
expect(redactTelemetryString("could not open ../-out.wav")).toBe("could not open [path]");
expect(redactTelemetryString("could not open .\\tmp\\-x.aac")).toBe("could not open [path]");
});

it("redacts a bare basename — a caller may pass one instead of a path", () => {
expect(redactTelemetryString("Invalid data found in my-client-cut.mp4")).toBe(
"Invalid data found in [file]",
);
});

// Over-redaction is cheap; these are ordinary in ffprobe stderr and turning
// them into [path] would make a diagnostic string useless.
it.each(["N/A", "24/1", "Stream #0:0", "moov atom not found", "48000/1001"])(
"leaves %s alone",
(text) => {
expect(redactTelemetryString(`ffprobe: ${text}`)).toBe(`ffprobe: ${text}`);
},
);

// A `?` is illegal in a Windows filename, so this is not a query string —
// the whole token is path, and must not survive by hiding behind a `?`.
it("consumes the rest of the token once a path is established", () => {
expect(redactTelemetryString("Navigation failed for C:\\Users\\A\\v.mov?not-a-query")).toBe(
"Navigation failed for [path]",
);
});

it("truncates after redacting, so a long path cannot survive by being cut", () => {
const out = redactTelemetryString(`/data/${"x".repeat(500)}/a.mp4`, 40);
expect(out).not.toContain("xxx");
});

// Named explicitly in review: a relative path with NO `./` prefix was
// missed by both the absolute rule (needs a leading slash) and the `./`
// rule (needs the dot), so it reached telemetry completely unredacted.
it.each([
"customer/acme-secret/video.mp4",
"assets/bgm.mp3",
"projects/client-name/cut/final.mov",
"a\\b\\c.wav",
])("redacts the bare relative path %s", (path) => {
const out = redactTelemetryString(`Invalid data found when processing ${path}`);
expect(out).toBe("Invalid data found when processing [path]");
});

it("redacts a dash-prefixed bare basename", () => {
expect(redactTelemetryString("could not open -customer-secret-intro.mp4")).toBe(
"could not open [file]",
);
});
});

// The segment classes were ASCII `\w`, so a non-Latin path went out verbatim.
// This redactor also feeds CLI telemetry and producer observation messages,
// where no known-path list is supplied to compensate.
describe("non-Latin paths", () => {
it.each([
"/数据/客户/秘密视频.mp4",
"/данные/клиент/видео.mp4",
"/data/客户/secret.mp4",
"/Users/alice/проект/видео.mp4",
])("redacts the absolute path %s", (path) => {
const out = redactTelemetryString(`ffprobe failed reading ${path}`);
expect(out).toBe("ffprobe failed reading [path]");
});

it("redacts a non-Latin bare relative path without stranding the first segment", () => {
// The lookbehind used to be `\w`-based, so a match could start mid-token
// when the preceding character was non-ASCII: this redacted to `客户[path]`.
expect(redactTelemetryString("could not open 客户/秘密/视频.mp4")).toBe(
"could not open [path]",
);
});

it.each(["./资产/背景.mp3", "../输出/final.wav"])("redacts the relative path %s", (path) => {
expect(redactTelemetryString(`could not open ${path}`)).toBe("could not open [path]");
});

it("redacts a non-Latin bare basename", () => {
expect(redactTelemetryString("Invalid data found in 秘密视频.mp4")).toBe(
"Invalid data found in [file]",
);
});
});

describe("redactKnownPaths", () => {
// Shape matching has holes by construction. A caller that built the argv
// knows the exact path, so it can name it instead of hoping a regex does.
it("redacts an exact path a regex would not recognise as one", () => {
const weird = "acme_secret_project";
expect(redactKnownPaths(`ffprobe: ${weird}: Invalid data`, [weird])).toBe(
"ffprobe: [path]: Invalid data",
);
});

it("redacts the basename too — ffprobe often reports only that", () => {
const out = redactKnownPaths("moov atom not found in secret-cut.mp4", [
"/data/x/secret-cut.mp4",
]);
expect(out).toContain("[path]");
expect(out).not.toContain("secret-cut");
});

it("leaves the message alone when no path was supplied", () => {
expect(redactKnownPaths("moov atom not found", [])).toBe("moov atom not found");
});

// Guards against a one/two-character basename turning every occurrence of
// that letter into [path].
it("ignores paths too short to be distinctive", () => {
expect(redactKnownPaths("a stream at a rate", ["a"])).toBe("a stream at a rate");
});

// This sits on an error path: throwing here turns a reported failure into an
// unhandled rejection, which is what a non-Error rejection's `undefined`
// message caused.
it.each([undefined, null, 42, {}])("returns a string for the non-string input %s", (value) => {
expect(() => redactKnownPaths(value as unknown as string, ["/tmp/x.mp4"])).not.toThrow();
});
});
140 changes: 134 additions & 6 deletions packages/core/src/telemetryRedaction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,141 @@ function redactUrlQueryStrings(value: string): string {
return value.replace(/\b(https?:\/\/[^\s?]+)\?[^\s]*/g, "$1?…");
}

/**
* Path characters we treat as part of a single segment. Space is deliberately
* excluded: including it would let a match run past the path and swallow the
* prose after it, and a path with a space still gets its remaining segments
* redacted, which is the part that carries the identifying information.
*/
/**
* A path segment is defined by what ENDS it, not by an alphabet.
*
* `[\w...]` is ASCII-only, so `/数据/客户/秘密视频.mp4` and `/data/客户/secret.mp4`
* passed through completely unredacted — the generic redactor also feeds CLI
* telemetry and producer observation messages, where no known-path list is
* supplied to cover for it. Enumerating Unicode classes instead (`\p{L}\p{N}…`)
* would work but has to be kept correct for marks, joiners and emoji; a
* delimiter-based rule is right for every script by construction.
*
* Whitespace and quotes end a segment; so do the separators themselves.
* Space stays excluded for the original reason: including it would let a match
* run past the path and swallow the prose after it.
*/
const SEGMENT = String.raw`[^\s/\\'"]+`;

/** Same, minus the dot, so a trailing `.ext` can be matched separately. */
const SEGMENT_NODOT = String.raw`[^\s/\\'".]+`;

/**
* Once a match is established as a path, consume the rest of the token.
* Windows forbids `?` in a filename, so `video.mov?not-a-query` is not a real
* query string — but stopping at the `?` would emit the remainder verbatim.
* Redacting to the next delimiter cannot leak; stopping early can.
*/
const TOKEN_TAIL = String.raw`[^\s'")]*`;

/**
* Absolute path, any root — NOT an allowlist of roots.
*
* The previous version enumerated `/Users`, `/home`, `/opt`, `/tmp`… which
* meant a project on `/data`, `/Volumes/External`, an NFS mount or any root a
* user invented reached telemetry verbatim. Two or more segments are required
* so `N/A` and a `24/1` frame rate — both ordinary in ffprobe stderr — are not
* mistaken for paths.
*
* The lookbehind keeps this off URLs: after `https:` the slash is preceded by
* `:`, the second by `/`, and the path segment by a word character, so no
* position inside a URL can start a match. URLs are handled above, where the
* host is kept and only the query is dropped.
*/
const ABSOLUTE_PATH = new RegExp(
String.raw`(?<![:\w/\\])(?:[A-Za-z]:)?(?:[\\/]${SEGMENT}){2,}${TOKEN_TAIL}`,
"g",
);

/** `./assets/bgm.mp3`, `../out.wav`, `.\tmp\x` — relative paths leak the same
* project structure absolute ones do, and were previously untouched. */
const RELATIVE_PATH = new RegExp(
String.raw`(?<![\w/\\.])\.{1,2}(?:[\\/]${SEGMENT})+${TOKEN_TAIL}`,
"g",
);

/**
* A bare basename with an asset extension. ffprobe reports the input by the
* name it was given, so a caller that passes a basename (or a path this
* flattened to its last segment) still names the user's file.
*
* The lookbehind excludes a slash so this cannot re-redact the tail of a URL
* whose host we deliberately keep.
*/
const ASSET_BASENAME =
/(?<![\w/\\])[^\s/\\'"]+\.(?:mp4|mov|mkv|webm|avi|m4v|mpe?g|ts|mp3|wav|aac|m4a|flac|ogg|opus|png|jpe?g|gif|webp|svg|html?|json|srt|vtt|ass)\b/gi;

/**
* A relative path with NO `./` prefix — `assets/bgm.mp3`,
* `customer/acme-secret/video.mp4`. These leak exactly as much as an absolute
* path and were missed by both rules above: the absolute rule requires a
* leading slash, and the `./` rule requires the dot.
*
* Qualifying needs either two separators or one plus a file extension, so the
* ordinary non-paths in ffprobe stderr — `N/A`, a `24/1` frame rate,
* `48000/1001` — do not match. The lookbehind keeps it off URL paths, whose
* host is deliberately kept.
*/
const BARE_RELATIVE_PATH = new RegExp(
[
// Two or more separators: `customer/acme/video.mp4`. No extension needed —
// that much structure is already a path.
String.raw`(?<![^\s'\"(=,\[])(?:${SEGMENT}[\\/]){2,}${SEGMENT}${TOKEN_TAIL}`,
// One separator, but the last segment carries a file extension:
// `assets/bgm.mp3`. That segment is dot-free on purpose — SEGMENT includes
// `.`, so a greedy one swallows the extension this rule needs.
String.raw`(?<![^\s'\"(=,\[])${SEGMENT}[\\/]${SEGMENT_NODOT}\.\w{1,8}\b${TOKEN_TAIL}`,
].join("|"),
"g",
);

/**
* Redact literal strings — the exact paths a caller KNOWS it passed — before
* any shape-based rule runs.
*
* Shape matching is a net with holes by construction; this is not. When the
* caller has the path in hand (it built the argv), spell it out rather than
* hoping a regex recognises it, and take the basename too since ffprobe often
* reports only that.
*/
export function redactKnownPaths(value: string, paths: readonly string[]): string {
// Fail soft on a non-string. This sits on an error path, so throwing here
// converts a reported failure into an unhandled rejection — which is exactly
// what happened when a caller passed a non-Error rejection's `.message`.
if (typeof value !== "string") return "";
let out = value;
for (const path of paths) {
if (typeof path !== "string" || path.length === 0) continue;
// Longest first: replacing the basename before the full path would leave
// the directory prefix stranded.
const basename = path.split(/[\\/]/).pop() ?? "";
for (const literal of [path, basename].filter((v) => v.length > 2)) {
out = out.split(literal).join("[path]");
}
}
return out;
}

function redactFilePaths(value: string): string {
return value
.replace(/file:\/\/[^\s'")]+/g, "[file-url]")
.replace(/\/Users\/[^\s'")]+/g, "[path]")
.replace(/\/(?:home|root|opt|app|workspace|srv|mnt)\/[^\s'")]+/g, "[path]")
.replace(/\/(?:private\/)?(?:var|tmp)\/[^\s'")]+/g, "[path]")
.replace(/[A-Za-z]:\\[^\s'")]+/g, "[path]");
return (
value
.replace(/file:\/\/[^\s'")]+/g, "[file-url]")
// Relative BEFORE absolute: `./assets/x.mp3` has an absolute-looking tail
// (`/assets/x.mp3`), so the absolute rule would consume it and leave the
// leading `.` stranded outside the redaction.
.replace(RELATIVE_PATH, "[path]")
// Bare-relative BEFORE absolute: a bare path's interior satisfies the
// absolute rule, which would claim it and strand the first segment.
.replace(BARE_RELATIVE_PATH, "[path]")
.replace(ABSOLUTE_PATH, "[path]")
.replace(ASSET_BASENAME, "[file]")
);
}

export function redactTelemetryString(
Expand Down
1 change: 1 addition & 0 deletions packages/lint/src/hevcPreviewLint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ async function probeIsHevc(ffprobePath: string, filePath: string): Promise<boole
"stream=codec_name",
"-of",
"json",
"--",
filePath,
]);
return hasHevcStream(JSON.parse(stdout));
Expand Down
1 change: 1 addition & 0 deletions packages/producer/src/plan-parity-analysis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ function probeMetadata(outputPath: string): PlanParityStreamMetadata {
].join(":"),
"-of",
"json",
"--",
outputPath,
]);
return normalizeFfprobeMetadata(JSON.parse(bytes.toString("utf-8")) as unknown);
Expand Down
2 changes: 2 additions & 0 deletions packages/producer/src/services/distributed/assemble.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ function probeStream(
"-count_packets",
"-of",
"json",
"--",
outputPath,
],
{ stdio: "pipe" },
Expand Down Expand Up @@ -418,6 +419,7 @@ describe("assemble()", () => {
"stream=r_frame_rate,avg_frame_rate,duration",
"-of",
"json",
"--",
outputPath,
],
{ stdio: "pipe" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ describe.skipIf(!hasFfmpeg)("audio pad real-media packet contract", () => {
"packet=duration_time",
"-of",
"json",
"--",
output,
],
{ encoding: "utf8" },
Expand Down
Loading
Loading