Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 49 additions & 2 deletions packages/cli/src/whisper/transcribe.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import { describe, expect, test } from "vitest";
import { dtwPresetForModel } from "./transcribe.js";
import { describe, expect, it, test } from "vitest";
import {
dtwPresetForModel,
resolveAudioPreparationTimeoutMs,
resolveWhisperTimeoutMs,
} from "./transcribe.js";

describe("dtwPresetForModel", () => {
// The large family is the regression: model files are hyphenated but
Expand All @@ -22,3 +26,46 @@ describe("dtwPresetForModel", () => {
},
);
});

describe("resolveWhisperTimeoutMs", () => {
it("keeps the existing five-minute floor for short recordings", () => {
expect(resolveWhisperTimeoutMs(10)).toBe(300_000);
});

it("scales the timeout for long recordings", () => {
expect(resolveWhisperTimeoutMs(41 * 60)).toBe(24_600_000);
});

it("caps the safety window at twelve hours", () => {
expect(resolveWhisperTimeoutMs(24 * 60 * 60)).toBe(43_200_000);
});

it("falls back to five minutes when duration is unavailable", () => {
expect(resolveWhisperTimeoutMs(null)).toBe(300_000);
expect(resolveWhisperTimeoutMs(Number.NaN)).toBe(300_000);
});

it.each([
[30, 300_000],
[30.1, 301_000],
[4319, 43_190_000],
[4320, 43_200_000],
])("clamps duration %ss to %sms", (duration, expected) => {
expect(resolveWhisperTimeoutMs(duration)).toBe(expected);
});
});

describe("resolveAudioPreparationTimeoutMs", () => {
it.each([
[10, 120_000],
[6 * 60 * 60, 10_800_000],
[24 * 60 * 60, 21_600_000],
])("scales duration %ss to %sms", (duration, expected) => {
expect(resolveAudioPreparationTimeoutMs(duration)).toBe(expected);
});

it("falls back to two minutes when duration is unavailable", () => {
expect(resolveAudioPreparationTimeoutMs(null)).toBe(120_000);
expect(resolveAudioPreparationTimeoutMs(Number.NaN)).toBe(120_000);
});
});
94 changes: 91 additions & 3 deletions packages/cli/src/whisper/transcribe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,84 @@ function findWavDataChunk(buf: Buffer): { offset: number; size: number } | null
return null;
}

const WHISPER_TIMEOUT_FLOOR_MS = 300_000;
const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000;
const WHISPER_TIMEOUT_CAP_MS = 43_200_000;
const AUDIO_PREPARATION_TIMEOUT_FLOOR_MS = 120_000;
const AUDIO_PREPARATION_TIMEOUT_PER_MEDIA_SECOND_MS = 500;
const AUDIO_PREPARATION_TIMEOUT_CAP_MS = 21_600_000;

/**
* Give long recordings enough time to transcribe while retaining a bounded
* failure window. Short recordings keep the historical five-minute timeout.
*/
export function resolveWhisperTimeoutMs(durationSeconds: number | null): number {
if (durationSeconds === null || !Number.isFinite(durationSeconds) || durationSeconds <= 0) {
return WHISPER_TIMEOUT_FLOOR_MS;
}

return Math.min(
WHISPER_TIMEOUT_CAP_MS,
Math.max(
WHISPER_TIMEOUT_FLOOR_MS,
Math.ceil(durationSeconds * WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS),
),
);
}

/**
* Bound FFmpeg audio preparation while allowing long recordings to scale past
* the historical two-minute timeout. The half-realtime allowance is generous
* for audio-only extraction without inheriting Whisper's much larger window.
*/
export function resolveAudioPreparationTimeoutMs(durationSeconds: number | null): number {
if (durationSeconds === null || !Number.isFinite(durationSeconds) || durationSeconds <= 0) {
return AUDIO_PREPARATION_TIMEOUT_FLOOR_MS;
}

return Math.min(
AUDIO_PREPARATION_TIMEOUT_CAP_MS,
Math.max(
AUDIO_PREPARATION_TIMEOUT_FLOOR_MS,
Math.ceil(durationSeconds * AUDIO_PREPARATION_TIMEOUT_PER_MEDIA_SECOND_MS),
),
);
}

function getMediaDurationSeconds(filePath: string): number | null {
try {
const ffprobePath = findFFprobe();
if (!ffprobePath) return null;
const raw = execFileSync(
ffprobePath,
[
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
filePath,
],
{ encoding: "utf-8", timeout: 10_000 },
);
const durationSeconds = Number.parseFloat(raw.trim());
return Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : null;
} catch {
return null;
}
}

function getPreparedWavDurationSeconds(wavPath: string): number | null {
try {
const dataChunk = findWavDataChunk(readFileSync(wavPath));
if (!dataChunk) return null;
return dataChunk.size / (16_000 * 2);
} catch {
return null;
}
}

/**
* Detect when speech begins in a 16kHz mono WAV by finding the first
* sustained energy jump above the track's median RMS. Returns onset time in
Expand Down Expand Up @@ -152,7 +230,10 @@ function extractAudio(videoPath: string): string {
execFileSync(
ffmpegPath,
["-i", videoPath, "-vn", "-ar", "16000", "-ac", "1", "-f", "wav", "-y", wavPath],
{ stdio: "ignore", timeout: 120_000 },
{
stdio: "ignore",
timeout: resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(videoPath)),
},
);
return wavPath;
}
Expand Down Expand Up @@ -200,7 +281,10 @@ function prepareAudio(audioPath: string): string {
execFileSync(
ffmpegPath,
["-i", audioPath, "-ar", "16000", "-ac", "1", "-f", "wav", "-y", wavPath],
{ stdio: "ignore", timeout: 120_000 },
{
stdio: "ignore",
timeout: resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(audioPath)),
},
);
return wavPath;
}
Expand Down Expand Up @@ -304,7 +388,11 @@ export async function transcribe(
}
whisperArgs.push(wavPath);

execFileSync(whisper.executablePath, whisperArgs, { stdio: "ignore", timeout: 300_000 });
const whisperTimeoutMs = resolveWhisperTimeoutMs(getPreparedWavDurationSeconds(wavPath));
execFileSync(whisper.executablePath, whisperArgs, {
stdio: "ignore",
timeout: whisperTimeoutMs,
});

// 6. Read and validate output
const transcriptPath = `${outputBase}.json`;
Expand Down
2 changes: 1 addition & 1 deletion skills-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"files": 10
},
"media-use": {
"hash": "a842eab0c9d3a0c0",
"hash": "11f3025c7c97dd8f",
"files": 124
},
"motion-graphics": {
Expand Down
13 changes: 5 additions & 8 deletions skills/media-use/scripts/resolve.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,7 @@ import {
flushHeygenFailureTracking,
versionLessThan,
} from "./lib/heygen-cli.mjs";
import {
BundledSfxAssetsError,
inspectBundledSfxAssets,
} from "./lib/bundled-sfx-provider.mjs";
import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs";

const INGEST_TYPES = [...listTypes(), "video"];

Expand Down Expand Up @@ -387,10 +384,10 @@ async function run() {
providerFailure instanceof BundledSfxAssetsError
? providerFailure.message
: type === "brand"
? "no brand spec found — add a frame.md or design.md (colors/font/logo) to this project. Run the HyperFrames design flow to create one; brand tokens are read locally for deterministic rendering."
: args.provider
? `provider "${args.provider}" could not resolve ${type}: "${intent}"${localOnly ? " (--local-only skips network providers; drop it or the --provider override)" : ""}`
: `no provider could resolve ${type}: "${intent}"`;
? "no brand spec found — add a frame.md or design.md (colors/font/logo) to this project. Run the HyperFrames design flow to create one; brand tokens are read locally for deterministic rendering."
: args.provider
? `provider "${args.provider}" could not resolve ${type}: "${intent}"${localOnly ? " (--local-only skips network providers; drop it or the --provider override)" : ""}`
: `no provider could resolve ${type}: "${intent}"`;
if (args.json) {
console.log(
JSON.stringify({
Expand Down
Loading