Podcast to social clips

"""A podcast episode to vertical social clips: transcript, highlight picks, captions, cover art, a spoken intro, a moving background.

pip install openai requests      (and ffmpeg on PATH)
RUNCRATE_API_KEY=... python podcast_to_clips.py episode.mp3 "Show name"
"""
import base64, json, os, subprocess, sys, time
from pathlib import Path

import requests
from openai import OpenAI

API = "https://api.runcrate.ai/v1"
KEY = os.environ["RUNCRATE_API_KEY"]
client = OpenAI(base_url=API, api_key=KEY, max_retries=4, timeout=180)
ASR = "openai/whisper-large-v3-turbo"           # segment timestamps
EDITOR = "openai/gpt-oss-120b"                  # on the free tier
ART = "black-forest-labs/FLUX-2-klein-4b"
TTS, VOICE = "hexgrad/Kokoro-82M", "am_michael"
MOTION = "bytedance/seedance-2.0-fast"          # 4 s at 480p, looped under each clip
OUT = Path("out"); OUT.mkdir(exist_ok=True)
ledger = []

def log(step, model, t0, cost):
    ledger.append({"step": step, "model": model, "seconds": round(time.time() - t0, 2), "cost_usd": cost})
    print(f"{step:<12} {model:<36} {time.time() - t0:7.2f}s  ${cost:.6f}")

def ffmpeg(*args):
    subprocess.run(["ffmpeg", "-y", "-loglevel", "error", *args], check=True)

def srt_time(t: float) -> str:
    return f"{int(t // 3600):02d}:{int(t % 3600 // 60):02d}:{int(t % 60):02d},{int(t * 1000 % 1000):03d}"

def video(body: dict, path: Path) -> float:
    """Submit, poll with backoff (riding out 429s and 5xx), download. Returns the billed cost."""
    h = {"Authorization": f"Bearer {KEY}"}
    for attempt in range(5):
        res = requests.post(f"{API}/videos", json=body, headers=h, timeout=60)
        if res.status_code not in (429, 500, 502, 503, 504):
            break
        time.sleep(int(res.headers.get("Retry-After", 2 ** attempt)))
    res.raise_for_status()
    job, cost, wait = res.json(), float(res.headers.get("x-llm-cost", 0)), 5
    while job["status"] not in ("completed", "failed"):
        time.sleep(wait)
        r = requests.get(f"{API}/videos/{job['id']}", headers=h, timeout=30)
        if r.status_code in (429, 500, 502, 503, 504):
            wait = min(wait * 2, 30); continue
        r.raise_for_status()
        job, wait = r.json(), min(wait * 1.5, 20)
    if job["status"] == "failed":
        raise RuntimeError(f"video {job['id']} failed: {job.get('error')}")
    path.write_bytes(requests.get(f"{API}/videos/{job['id']}/download", headers=h, timeout=300).content)
    return cost

def main(episode: str, show: str) -> None:
    t0 = time.time()
    with open(episode, "rb") as f:
        raw = client.audio.transcriptions.with_raw_response.create(model=ASR, file=f, response_format="verbose_json")
    segs = [{"i": i, "start": s.start, "end": s.end, "text": s.text.strip()} for i, s in enumerate(raw.parse().segments)]
    log("transcribe", ASR, t0, float(raw.headers.get("x-llm-cost", 0)))

    t0 = time.time()
    raw = client.chat.completions.with_raw_response.create(
        model=EDITOR, temperature=0,
        messages=[{"role": "user", "content":
            "Pick the two best moments for short social clips. Each is a run of whole segments, 12 to 40 seconds long, "
            "that makes sense on its own. Also write one cover-art prompt for the episode (no text in the image).\n\n"
            + "\n".join(f"{s['i']} [{s['start']:.1f}-{s['end']:.1f}] {s['text']}" for s in segs)}],
        response_format={"type": "json_schema", "json_schema": {"name": "picks", "strict": True, "schema": {
            "type": "object", "additionalProperties": False, "required": ["clips", "cover_prompt"],
            "properties": {
                "cover_prompt": {"type": "string"},
                "clips": {"type": "array", "items": {"type": "object", "additionalProperties": False,
                          "required": ["first_segment", "last_segment", "title", "hook"],
                          "properties": {"first_segment": {"type": "integer"}, "last_segment": {"type": "integer"},
                                         "title": {"type": "string"}, "hook": {"type": "string", "description": "One spoken sentence, at most 12 words"}}}},
            }}}},
    )
    picks = json.loads(raw.parse().choices[0].message.content)
    log("pick", EDITOR, t0, float(raw.headers.get("x-llm-cost", 0)))

    t0 = time.time()
    raw = client.images.with_raw_response.generate(model=ART, prompt=picks["cover_prompt"], extra_body={"aspect_ratio": "9:16"})
    url = raw.parse().data[0].url
    (OUT / "cover.jpg").write_bytes(base64.b64decode(url.split(",", 1)[1]) if url.startswith("data:") else requests.get(url, timeout=60).content)
    log("cover", ART, t0, float(raw.headers.get("x-llm-cost", 0)))

    t0 = time.time()
    cover = base64.b64encode((OUT / "cover.jpg").read_bytes()).decode()
    cost = video({"model": MOTION, "prompt": "slow parallax drift, gentle light movement, seamless", "duration": 4,
                  "resolution": "480p", "aspect_ratio": "9:16", "generate_audio": False, "image": f"data:image/jpeg;base64,{cover}"},
                 OUT / "background.mp4")
    log("background", MOTION, t0, cost)

    burn = "subtitles" in subprocess.run(["ffmpeg", "-hide_banner", "-filters"], capture_output=True, text=True).stdout
    for n, clip in enumerate(picks["clips"][:2], 1):
        first, last = segs[clip["first_segment"]], segs[min(clip["last_segment"], len(segs) - 1)]
        start, end = first["start"], last["end"]

        t0 = time.time()
        raw = client.audio.speech.with_raw_response.create(model=TTS, voice=VOICE, input=f"From {show}. {clip['hook']}", response_format="wav")
        (OUT / f"intro-{n}.wav").write_bytes(raw.content)
        log(f"intro {n}", TTS, t0, float(raw.headers.get("x-llm-cost", 0)))

        t0 = time.time()
        intro = float(subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(OUT / f"intro-{n}.wav")],
                                     capture_output=True, text=True).stdout)
        lines = [s for s in segs if s["end"] > start and s["start"] < end]
        srt = "".join(f"{k}\n{srt_time(max(s['start'], start) - start + intro)} --> {srt_time(min(s['end'], end) - start + intro)}\n{s['text']}\n\n"
                      for k, s in enumerate(lines, 1))
        (OUT / f"clip-{n}.srt").write_text(srt)
        # Intro voice, then the cut, over the looping background, 1080x1920.
        ffmpeg("-i", str(OUT / f"intro-{n}.wav"), "-ss", f"{start}", "-to", f"{end}", "-i", episode,
               "-filter_complex", "[0:a][1:a]concat=n=2:v=0:a=1[a]", "-map", "[a]", str(OUT / f"audio-{n}.wav"))
        vf = "scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920"
        if burn:
            vf += f",subtitles={OUT / f'clip-{n}.srt'}:force_style='Fontsize=14,MarginV=60'"
        ffmpeg("-stream_loop", "-1", "-i", str(OUT / "background.mp4"), "-i", str(OUT / f"audio-{n}.wav"), "-i", str(OUT / f"clip-{n}.srt"),
               "-map", "0:v", "-map", "1:a", "-map", "2:s", "-vf", vf, "-shortest",
               "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-c:s", "mov_text", str(OUT / f"clip-{n}.mp4"))
        log(f"cut {n}", "ffmpeg" + (" (burned captions)" if burn else " (caption track + .srt)"), t0, 0.0)
        print(f"  clip {n}: {clip['title']}  {end - start:.1f}s from {start:.1f}s")

    total = sum(s["cost_usd"] for s in ledger)
    Path("run.json").write_text(json.dumps({"picks": picks, "steps": ledger, "total_usd": total}, indent=2))
    print(f"\ntotal ${total:.4f}")

if __name__ == "__main__":
    main(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "the show")

// A podcast episode to vertical social clips: transcript, highlight picks, captions, cover art, a spoken intro, a moving background.
//
// npm install openai      (and ffmpeg on PATH)
// RUNCRATE_API_KEY=... npx tsx podcast-to-clips.ts episode.mp3 "Show name"
import { execFileSync } from "node:child_process";
import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import OpenAI from "openai";

const API = "https://api.runcrate.ai/v1";
const KEY = process.env.RUNCRATE_API_KEY ?? "";
const client = new OpenAI({ baseURL: API, apiKey: KEY, maxRetries: 4, timeout: 180_000 });
const ASR = "openai/whisper-large-v3-turbo"; // segment timestamps
const EDITOR = "openai/gpt-oss-120b"; // on the free tier
const ART = "black-forest-labs/FLUX-2-klein-4b";
const TTS = "hexgrad/Kokoro-82M";
const VOICE = "am_michael";
const MOTION = "bytedance/seedance-2.0-fast"; // 4 s at 480p, looped under each clip
const ledger: { step: string; model: string; seconds: number; cost_usd: number }[] = [];
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
const ffmpeg = (...args: string[]) => execFileSync("ffmpeg", ["-y", "-loglevel", "error", ...args]);
mkdirSync("out", { recursive: true });

function log(step: string, model: string, t0: number, cost: number) {
  const seconds = (Date.now() - t0) / 1000;
  ledger.push({ step, model, seconds, cost_usd: cost });
  console.log(`${step.padEnd(12)} ${model.padEnd(36)} ${seconds.toFixed(2).padStart(7)}s  $${cost.toFixed(6)}`);
}

function srtTime(t: number): string {
  const ms = Math.round(t * 1000);
  const p = (n: number, w = 2) => String(n).padStart(w, "0");
  return `${p(Math.floor(ms / 3_600_000))}:${p(Math.floor((ms % 3_600_000) / 60_000))}:${p(Math.floor((ms % 60_000) / 1000))},${p(ms % 1000, 3)}`;
}

/** Submit, poll with backoff (riding out 429s and 5xx), download. Returns the billed cost. */
async function video(body: object, path: string): Promise<number> {
  const h = { Authorization: `Bearer ${KEY}`, "Content-Type": "application/json" };
  let res = await fetch(`${API}/videos`, { method: "POST", headers: h, body: JSON.stringify(body) });
  for (let attempt = 0; [429, 500, 502, 503, 504].includes(res.status) && attempt < 5; attempt++) {
    await sleep(Number(res.headers.get("retry-after") ?? 2 ** attempt) * 1000);
    res = await fetch(`${API}/videos`, { method: "POST", headers: h, body: JSON.stringify(body) });
  }
  if (!res.ok) throw new Error(`video submit answered ${res.status}: ${await res.text()}`);
  const cost = Number(res.headers.get("x-llm-cost") ?? 0);
  let job = (await res.json()) as { id: string; status: string; error?: unknown };
  let wait = 5000;
  while (job.status !== "completed" && job.status !== "failed") {
    await sleep(wait);
    const r = await fetch(`${API}/videos/${job.id}`, { headers: h });
    if ([429, 500, 502, 503, 504].includes(r.status)) {
      wait = Math.min(wait * 2, 30_000);
      continue;
    }
    if (!r.ok) throw new Error(`video poll answered ${r.status}`);
    job = (await r.json()) as typeof job;
    wait = Math.min(wait * 1.5, 20_000);
  }
  if (job.status === "failed") throw new Error(`video ${job.id} failed: ${JSON.stringify(job.error)}`);
  writeFileSync(path, Buffer.from(await (await fetch(`${API}/videos/${job.id}/download`, { headers: h })).arrayBuffer()));
  return cost;
}

const [episode, show = "the show"] = process.argv.slice(2);

let t0 = Date.now();
const tr = await client.audio.transcriptions.create({ model: ASR, file: createReadStream(episode), response_format: "verbose_json" }).withResponse();
const segs = (tr.data.segments ?? []).map((s, i) => ({ i, start: s.start, end: s.end, text: s.text.trim() }));
log("transcribe", ASR, t0, Number(tr.response.headers.get("x-llm-cost") ?? 0));

t0 = Date.now();
type Picks = { cover_prompt: string; clips: { first_segment: number; last_segment: number; title: string; hook: string }[] };
const pick = await client.chat.completions
  .create({
    model: EDITOR,
    temperature: 0,
    messages: [
      {
        role: "user",
        content:
          "Pick the two best moments for short social clips. Each is a run of whole segments, 12 to 40 seconds long, " +
          "that makes sense on its own. Also write one cover-art prompt for the episode (no text in the image).\n\n" +
          segs.map((s) => `${s.i} [${s.start.toFixed(1)}-${s.end.toFixed(1)}] ${s.text}`).join("\n"),
      },
    ],
    response_format: {
      type: "json_schema",
      json_schema: {
        name: "picks",
        strict: true,
        schema: {
          type: "object",
          additionalProperties: false,
          required: ["clips", "cover_prompt"],
          properties: {
            cover_prompt: { type: "string" },
            clips: {
              type: "array",
              items: {
                type: "object",
                additionalProperties: false,
                required: ["first_segment", "last_segment", "title", "hook"],
                properties: {
                  first_segment: { type: "integer" },
                  last_segment: { type: "integer" },
                  title: { type: "string" },
                  hook: { type: "string", description: "One spoken sentence, at most 12 words" },
                },
              },
            },
          },
        },
      },
    },
  })
  .withResponse();
const picks: Picks = JSON.parse(pick.data.choices[0].message.content ?? "{}");
log("pick", EDITOR, t0, Number(pick.response.headers.get("x-llm-cost") ?? 0));

t0 = Date.now();
const art = await client.images.generate({ model: ART, prompt: picks.cover_prompt, aspect_ratio: "9:16" } as OpenAI.Images.ImageGenerateParamsNonStreaming).withResponse();
const url = art.data.data?.[0]?.url ?? "";
writeFileSync("out/cover.jpg", url.startsWith("data:") ? Buffer.from(url.split(",", 2)[1], "base64") : Buffer.from(await (await fetch(url)).arrayBuffer()));
log("cover", ART, t0, Number(art.response.headers.get("x-llm-cost") ?? 0));

t0 = Date.now();
const cover = readFileSync("out/cover.jpg").toString("base64");
const bgCost = await video(
  {
    model: MOTION,
    prompt: "slow parallax drift, gentle light movement, seamless",
    duration: 4,
    resolution: "480p",
    aspect_ratio: "9:16",
    generate_audio: false,
    image: `data:image/jpeg;base64,${cover}`,
  },
  "out/background.mp4",
);
log("background", MOTION, t0, bgCost);

const burn = execFileSync("ffmpeg", ["-hide_banner", "-filters"]).toString().includes(" subtitles ");
for (const [k, clip] of picks.clips.slice(0, 2).entries()) {
  const n = k + 1;
  const start = segs[clip.first_segment].start;
  const end = segs[Math.min(clip.last_segment, segs.length - 1)].end;

  t0 = Date.now();
  const speech = await client.audio.speech.create({ model: TTS, voice: VOICE, input: `From ${show}. ${clip.hook}`, response_format: "wav" }).withResponse();
  writeFileSync(`out/intro-${n}.wav`, Buffer.from(await speech.data.arrayBuffer()));
  log(`intro ${n}`, TTS, t0, Number(speech.response.headers.get("x-llm-cost") ?? 0));

  t0 = Date.now();
  const intro = Number(execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", `out/intro-${n}.wav`]).toString());
  const lines = segs.filter((s) => s.end > start && s.start < end);
  const srt = lines
    .map((s, j) => `${j + 1}\n${srtTime(Math.max(s.start, start) - start + intro)} --> ${srtTime(Math.min(s.end, end) - start + intro)}\n${s.text}\n\n`)
    .join("");
  writeFileSync(`out/clip-${n}.srt`, srt);
  // Intro voice, then the cut, over the looping background, 1080x1920.
  ffmpeg("-i", `out/intro-${n}.wav`, "-ss", `${start}`, "-to", `${end}`, "-i", episode, "-filter_complex", "[0:a][1:a]concat=n=2:v=0:a=1[a]", "-map", "[a]", `out/audio-${n}.wav`);
  let vf = "scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920";
  if (burn) vf += `,subtitles=out/clip-${n}.srt:force_style='Fontsize=14,MarginV=60'`;
  ffmpeg(
    "-stream_loop", "-1", "-i", "out/background.mp4", "-i", `out/audio-${n}.wav`, "-i", `out/clip-${n}.srt`,
    "-map", "0:v", "-map", "1:a", "-map", "2:s", "-vf", vf, "-shortest",
    "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-c:s", "mov_text", `out/clip-${n}.mp4`,
  );
  log(`cut ${n}`, burn ? "ffmpeg (burned captions)" : "ffmpeg (caption track + .srt)", t0, 0);
  console.log(`  clip ${n}: ${clip.title}  ${(end - start).toFixed(1)}s from ${start.toFixed(1)}s`);
}

const total = ledger.reduce((s, r) => s + r.cost_usd, 0);
writeFileSync("run.json", JSON.stringify({ picks, steps: ledger, total_usd: total }, null, 2));
console.log(`\ntotal $${total.toFixed(4)}`);