> ## Documentation Index
> Fetch the complete documentation index at: https://hyperframes-feat-thread-message-stack.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.

# Media and audio

> Ask for the voiceover, music, sound, captions, cutouts, and assets a composition needs — with the precise, unambiguous phrasing the media pipeline acts on.

export const DocsVideo = ({src, poster, title, autoPlay = false, loop = false, portrait = false}) => {
  const videoRef = useRef(null);
  const playerRef = useRef(null);
  const hideTimerRef = useRef(null);
  const progressFrameRef = useRef(null);
  const [enhanced, setEnhanced] = useState(false);
  const [playing, setPlaying] = useState(false);
  const [waiting, setWaiting] = useState(false);
  const [muted, setMuted] = useState(false);
  const [currentTime, setCurrentTime] = useState(0);
  const [duration, setDuration] = useState(0);
  const [playbackRate, setPlaybackRate] = useState(1);
  const [controlsVisible, setControlsVisible] = useState(false);
  const [fullscreen, setFullscreen] = useState(false);
  const [fullscreenSupported, setFullscreenSupported] = useState(false);
  const [previewing, setPreviewing] = useState(false);
  const [scrubbing, setScrubbing] = useState(false);
  const [previewTime, setPreviewTime] = useState(0);
  const [previewPosition, setPreviewPosition] = useState(0);
  const formatTime = seconds => {
    if (!Number.isFinite(seconds) || seconds < 0) return "0:00";
    const minutes = Math.floor(seconds / 60);
    const remaining = Math.floor(seconds % 60);
    return `${minutes}:${String(remaining).padStart(2, "0")}`;
  };
  const clearHideTimer = () => {
    if (hideTimerRef.current) {
      window.clearTimeout(hideTimerRef.current);
      hideTimerRef.current = null;
    }
  };
  const revealControls = () => {
    setControlsVisible(true);
    clearHideTimer();
    hideTimerRef.current = window.setTimeout(() => setControlsVisible(false), 2200);
  };
  const togglePlayback = async () => {
    const video = videoRef.current;
    if (!video) return;
    if (video.paused || video.ended) {
      if (video.ended) video.currentTime = 0;
      setWaiting(true);
      try {
        await video.play();
      } catch {
        setWaiting(false);
        setPlaying(false);
      }
    } else {
      video.pause();
      setControlsVisible(true);
    }
  };
  const toggleMute = () => {
    const video = videoRef.current;
    if (!video) return;
    if (video.muted && video.volume === 0) video.volume = 0.8;
    video.muted = !video.muted;
    setMuted(video.muted);
  };
  const seek = event => {
    const video = videoRef.current;
    if (!video) return;
    const nextTime = Number(event.target.value);
    video.currentTime = nextTime;
    setCurrentTime(nextTime);
  };
  const updateScrubPreview = (event, seekMainVideo = false) => {
    if (!duration) return;
    const rect = event.currentTarget.getBoundingClientRect();
    const ratio = Math.min(1, Math.max(0, (event.clientX - rect.left) / rect.width));
    const nextTime = ratio * duration;
    setPreviewing(true);
    setPreviewTime(nextTime);
    setPreviewPosition(ratio * 100);
    if (seekMainVideo) {
      const video = videoRef.current;
      if (video) {
        video.currentTime = nextTime;
        setCurrentTime(nextTime);
      }
    }
  };
  const cyclePlaybackRate = () => {
    const video = videoRef.current;
    if (!video) return;
    const rates = [1, 1.25, 1.5, 2];
    const currentIndex = rates.indexOf(video.playbackRate);
    const nextRate = rates[(currentIndex + 1) % rates.length];
    video.playbackRate = nextRate;
    setPlaybackRate(nextRate);
  };
  const toggleFullscreen = async () => {
    const player = playerRef.current;
    const video = videoRef.current;
    if (!player || typeof document === "undefined") return;
    try {
      if (document.fullscreenElement) {
        await document.exitFullscreen();
      } else if (player.requestFullscreen) {
        await player.requestFullscreen();
      } else if (video?.webkitEnterFullscreen) {
        video.webkitEnterFullscreen();
      }
    } catch {}
  };
  const handleKeyboard = event => {
    if (event.target !== event.currentTarget) return;
    const video = videoRef.current;
    if (!video) return;
    if (event.key === " " || event.key === "Enter") {
      event.preventDefault();
      togglePlayback();
    } else if (event.key === "ArrowLeft") {
      event.preventDefault();
      video.currentTime = Math.max(0, video.currentTime - 5);
    } else if (event.key === "ArrowRight") {
      event.preventDefault();
      video.currentTime = Math.min(duration || video.duration || 0, video.currentTime + 5);
    } else if (event.key.toLowerCase() === "m") {
      event.preventDefault();
      toggleMute();
    } else if (event.key.toLowerCase() === "f") {
      event.preventDefault();
      toggleFullscreen();
    }
  };
  useEffect(() => {
    setEnhanced(true);
    setFullscreenSupported(Boolean(playerRef.current?.requestFullscreen || videoRef.current?.webkitEnterFullscreen));
    return () => {
      clearHideTimer();
    };
  }, []);
  useEffect(() => {
    if (typeof document === "undefined") return undefined;
    const syncFullscreen = () => setFullscreen(document.fullscreenElement === playerRef.current);
    document.addEventListener("fullscreenchange", syncFullscreen);
    return () => document.removeEventListener("fullscreenchange", syncFullscreen);
  }, []);
  useEffect(() => {
    clearHideTimer();
    if (!playing) return undefined;
    hideTimerRef.current = window.setTimeout(() => setControlsVisible(false), 2200);
    return clearHideTimer;
  }, [playing]);
  useEffect(() => {
    if (!playing) return undefined;
    const updateProgress = () => {
      const video = videoRef.current;
      if (video && !video.paused) setCurrentTime(video.currentTime);
      progressFrameRef.current = window.requestAnimationFrame(updateProgress);
    };
    progressFrameRef.current = window.requestAnimationFrame(updateProgress);
    return () => {
      if (progressFrameRef.current) window.cancelAnimationFrame(progressFrameRef.current);
      progressFrameRef.current = null;
    };
  }, [playing]);
  const progress = duration > 0 ? currentTime / duration * 100 : 0;
  const replaying = duration > 0 && currentTime >= duration - 0.15;
  return <div className="hf-docs-video-block" data-portrait={portrait ? "true" : "false"}>
      <div ref={playerRef} className="hf-docs-video" role="region" aria-label={title} tabIndex={0} onKeyDown={handleKeyboard} onPointerMove={revealControls} onPointerLeave={() => setControlsVisible(false)} onFocus={revealControls} onBlur={event => {
    if (!event.currentTarget.contains(event.relatedTarget)) setControlsVisible(false);
  }}>
        <video ref={videoRef} aria-label={title} src={src} poster={poster} autoPlay={autoPlay} loop={loop} playsInline preload="metadata" controls={!enhanced} onClick={togglePlayback} onDoubleClick={toggleFullscreen} onLoadedMetadata={event => {
    const nextDuration = event.currentTarget.duration || 0;
    setDuration(nextDuration);
    setMuted(event.currentTarget.muted);
  }} onDurationChange={event => setDuration(event.currentTarget.duration || 0)} onTimeUpdate={event => setCurrentTime(event.currentTarget.currentTime)} onPlay={() => setPlaying(true)} onPause={() => setPlaying(false)} onPlaying={() => setWaiting(false)} onWaiting={() => setWaiting(true)} onCanPlay={() => setWaiting(false)} onEnded={() => {
    setPlaying(false);
    setControlsVisible(true);
  }} onVolumeChange={event => setMuted(event.currentTarget.muted)} />

        {enhanced && <>
            {!playing && (currentTime <= 0.2 || replaying) && <button type="button" className="hf-docs-video-hero-play" onClick={togglePlayback} aria-label={replaying ? "Replay video" : "Play video"}>
                <span className="hf-docs-video-hero-icon" aria-hidden="true">
                  <svg viewBox="0 0 24 24">
                    <path d="M8 5.5v13l10-6.5z" />
                  </svg>
                </span>
              </button>}

            {waiting && playing && <span className="hf-docs-video-spinner" aria-label="Loading" />}

            <div className="hf-docs-video-controls" data-visible={controlsVisible ? "true" : "false"}>
              <div className="hf-docs-video-scrub-preview" data-visible={previewing ? "true" : "false"} style={{
    "--hf-video-preview-x": `${previewPosition}%`
  }} aria-hidden="true">
                <span>{formatTime(previewTime)}</span>
              </div>

              <input className="hf-docs-video-progress" type="range" min="0" max={duration || 0} step="0.01" value={Math.min(currentTime, duration || 0)} aria-label="Video progress" aria-valuetext={`${formatTime(currentTime)} of ${formatTime(duration)}`} onChange={seek} onPointerEnter={updateScrubPreview} onPointerMove={event => updateScrubPreview(event, scrubbing || event.buttons === 1)} onPointerDown={event => {
    setScrubbing(true);
    event.currentTarget.setPointerCapture?.(event.pointerId);
    updateScrubPreview(event, true);
  }} onPointerUp={event => {
    setScrubbing(false);
    if (event.pointerType !== "mouse") setPreviewing(false);
  }} onPointerCancel={() => {
    setScrubbing(false);
    setPreviewing(false);
  }} onPointerLeave={() => {
    if (!scrubbing) setPreviewing(false);
  }} style={{
    "--hf-video-progress": `${progress}%`
  }} />

              <div className="hf-docs-video-control-row">
                <button type="button" className="hf-docs-video-control" onClick={togglePlayback} aria-label={playing ? "Pause video" : "Play video"}>
                  {playing ? <svg viewBox="0 0 24 24" aria-hidden="true">
                      <path d="M7 5h4v14H7zm6 0h4v14h-4z" />
                    </svg> : <svg viewBox="0 0 24 24" aria-hidden="true">
                      <path d="M8 5.5v13l10-6.5z" />
                    </svg>}
                </button>

                <button type="button" className="hf-docs-video-control" onClick={toggleMute} aria-label={muted ? "Unmute video" : "Mute video"}>
                  {muted ? <svg viewBox="0 0 24 24" aria-hidden="true">
                      <path d="M4 9v6h4l5 4V5L8 9zm11.5 1.1 1.4-1.4 1.6 1.6 1.6-1.6 1.4 1.4-1.6 1.6 1.6 1.6-1.4 1.4-1.6-1.6-1.6 1.6-1.4-1.4 1.6-1.6z" />
                    </svg> : <svg viewBox="0 0 24 24" aria-hidden="true">
                      <path d="M4 9v6h4l5 4V5L8 9zm11 1.2v3.6c1-.5 1.7-1.5 1.7-2.8S16 10.7 15 10.2zm0-4v2.1c2.2.6 3.7 2.5 3.7 4.7s-1.5 4.1-3.7 4.7v2.1c3.3-.7 5.7-3.5 5.7-6.8S18.3 6.9 15 6.2z" />
                    </svg>}
                </button>

                <span className="hf-docs-video-time" aria-hidden="true">
                  {formatTime(currentTime)} <span>/</span> {formatTime(duration)}
                </span>

                <span className="hf-docs-video-spacer" />

                <button type="button" className="hf-docs-video-rate" onClick={cyclePlaybackRate} aria-label={`Playback speed ${playbackRate} times`}>
                  {playbackRate}×
                </button>

                {fullscreenSupported && <button type="button" className="hf-docs-video-control" onClick={toggleFullscreen} aria-label={fullscreen ? "Exit fullscreen" : "Enter fullscreen"}>
                    {fullscreen ? <svg viewBox="0 0 24 24" aria-hidden="true">
                        <path d="M8 3H6v3H3v2h5zm8 0v5h5V6h-3V3zM3 16v2h3v3h2v-5zm13 0v5h2v-3h3v-2z" />
                      </svg> : <svg viewBox="0 0 24 24" aria-hidden="true">
                        <path d="M3 8h2V5h3V3H3zm13-5v2h3v3h2V3zM5 16H3v5h5v-2H5zm14 3h-3v2h5v-5h-2z" />
                      </svg>}
                  </button>}
              </div>
            </div>
          </>}
      </div>

    </div>;
};

By now your video moves and reads right; this level gives it a voice. HyperFrames owns media *playback*; a sibling media pipeline resolves everything else — voice, music, sound effects, images, icons, logos, captions, and background removal. You reach all of it by describing what the composition needs, and the agent resolves each need to a frozen local file. The craft here is precision: vague media asks ("add some music," "no sound") are the ones that come back wrong, because the pipeline does exactly what the words say.

## Voiceover (TTS)

Text-to-speech runs locally through Kokoro — no API key needed — with a HeyGen TTS upsell behind it. Describe the content and the agent picks a fitting voice, or name the voice, tone, and speed directly:

> Generate narration for this script with a professional female voice.

> Add TTS voiceover, British male voice, at 1.1× speed.

The [Vocabulary](/prompting/vocabulary#text-to-speech-voices) table maps content types to Kokoro voices (for example `af_heart` / `af_nova` for a product demo, `am_adam` / `bf_emma` for a tutorial, `af_sky` / `am_michael` for marketing). Name one directly if you already know it; otherwise describe the read and let the agent choose.

* ❌ `add a voice`
* ✅ `warm, unhurried female narration of the quoted script` — tone and pace are what actually change the delivery

## Background music

Music resolves from a large catalog by mood, and it should almost always sit *under* the narration, not compete with it. Give the mood **and** a loudness target — the pipeline can duck and normalize to a level, so an explicit target lands a mix instead of a guess:

> Add subtle electronic BGM, kept under −18 dB so it stays beneath the voiceover.

> Upbeat tech-launch music bed at a low level, ducking under narration.

* ❌ `add background music` — you'll get a full-volume track fighting the VO
* ✅ `subtle background music, ducked ~12 dB under the voice` — a mix instruction the pipeline can execute

<Tip>
  A stated loudness target ("under −18 dB," "ducked under the voice") is the difference between music that supports the piece and music that buries it. When there's narration, always say the bed goes under it.
</Tip>

## Sound effects

SFX resolve from a small bundled library plus the catalog. Cue them to specific moments — a transition, a stamp-in, an impact — rather than sprinkling them:

> Add a whoosh on each of the three scene transitions.

> Put a soft click on the button press at 0:04.

## Pace reveals to the narration

Once a video has a voice, the voice is the clock — the next instruction that matters is telling the agent that on-screen elements land **on their spoken cues** — the stat appears as the narrator says it, not at some independent time the builder eyeballed. Without this, narration and visuals drift into two parallel tracks that happen to share a file:

> VO-paced reveals: each scene's elements land on their spoken cues; secondary elements keep resolving while the narrator is mid-thought; the scene is complete just as the narration moves on.

The capstone film applies exactly this rule to every region — its Direction block reads:

> VO-paced reveals: each region's elements land on their spoken cues as the camera arrives; secondary elements keep resolving while the camera is present; the region is complete just as the camera accelerates away.

Two practical notes: the agent gets word timings for free (the narration is transcribed with per-word timestamps, the same machinery behind [captions](#captions-and-transcription)), so "on its spoken cue" is a real, executable instruction — and the inverse rule matters just as much: the narration never waits for the visuals. Pace the camera and reveals to the voice, not the voice to the animation.

## Captions and transcription

Captions come from word-level timestamps. When you generate a voiceover, the timing comes with it; for existing footage, transcription produces the timing (Parakeet by default, with a whisper.cpp fallback). Scaffolding a project from a source video can generate captions from its audio directly.

> Transcribe the narration and add karaoke-style captions synced to it.

> Generate captions from `assets/interview.mp4` and style them hype, scale-pop.

Caption *look* is its own vocabulary (tone, size, per-word emphasis) — see [Captions catalog](/prompting/captions-catalog) for the styles. This page is about producing the timed text; that page is about styling it.

## Background removal (transparent cutouts)

The `remove-background` command mattes a subject out of a video or image locally and hands you a transparent WebM you can drop into any scene as a `<video>`:

> Remove the background from `assets/presenter.mp4` and float the subject over the scene.

One caveat is load-bearing: the built-in model is **purpose-built for people** — head-and-shoulders or full-body, reasonably stable framing, a background that contrasts with the subject. It returns a mostly-empty mask on **non-human subjects** (products, animals, objects). If you need to cut out a product, say so — the agent should route to a different tool rather than run the person model and get nothing.

* ❌ `remove the background from this product shot` with the built-in command — the human-matting model can't see it
* ✅ `matte the presenter out of assets/talk.mp4` (person) — or, for a product, flag that it's a non-human subject so a different matter is used

The [Remove background guide](/guides/remove-background) covers the person-only caveat, the two-layer plate for text-behind-subject, and alternatives for objects and hair-fine mattes.

## Video-in-video and picture-in-picture

Layering footage — a talking head over a scene, a subject in front of a headline, PiP inset — is a compositing prompt. Two grounded rules keep it frame-accurate, and the agent applies them for you, but naming the layout you want helps:

> Put the transparent presenter cutout in the bottom-right, over the chart scene.

> Layer the headline *behind* the presenter so their silhouette occludes the text.

<Note>
  Two mechanics the workflow skills handle automatically (from the [Remove background guide](/guides/remove-background#compositing-patterns-and-pitfalls)): a cutout that reveals into view is wrapped in a non-timed `<div>` and the *wrapper* is animated (the framework forces `opacity: 1` on timed clips, so animating the video directly does nothing); and both the base video and the cutout mount at `data-start="0"` so their decoders stay in sync at the cut. You rarely need to say this — but it's why "late-mounting" a PiP clip can land a frame off.
</Note>

## Bring any footage

You don't need to pre-convert supplied footage before naming it in a prompt. If a clip's codec doesn't play back cleanly in a browser — HEVC (H.265) is the common case, straight off an iPhone or a screen recorder — the framework probes the asset and builds a bounded H.264 proxy automatically, cached under `.transcode-cache/`. `preview`, `play`, Studio, and published player pages use the proxy for playback; a render always decodes the original file, so nothing about final quality or color is touched. `hyperframes lint` also flags the asset at info level (`hevc_preview_codec`) so you know a proxy is in play, and it's optional — `--no-proxy` per command, or `media.autoProxy: false` in `hyperframes.json` project-wide. The same mechanism covers alpha-channel sources too (ProRes 4444, alpha WebM proxy to VP9+Opus WebM instead of being refused), so a transparent cutout in a hostile codec isn't a blocker either.

None of this changes how you phrase the ask: name the footage by path like any other supplied asset, and describe the composition you want built from it.

> Build a short picture-in-picture piece from `source-hevc.mp4` — inset it bottom-right over a full-bleed background scene, with a soft rounded border.

<DocsVideo title="HyperFrames video: Proxy Footage" src="https://static.heygen.ai/hyperframes-oss/docs/images/prompting/proxy-footage.mp4#t=0.1" loop />

*Rendered from the prompt above, unedited — the source clip is a plain H.265/HEVC file; render decoded it directly via FFmpeg, while preview would have used the automatic H.264 proxy.*

See the [Rendering guide](/guides/rendering#input-video-codecs) for the mechanics — proxy generation, caching, and which codecs it covers.

## The supplied-assets rule

For any asset you already have, an explicit path is the instruction that removes the most ambiguity. The agent will search when you describe an asset, but a path removes every ambiguity about *which* file — and for your own brand assets, it's the only way to guarantee the right one:

* ❌ `use my logo`
* ✅ `use assets/logo.svg`

This matters even when resolution would otherwise work: brand and entity assets should point at *your* file, not a resolved lookalike. (Third-party logos are a separate case — the pipeline pulls official marks from a logo cascade and never hand-redraws them, so "add the LinkedIn logo" is fine; "add my company's logo" needs a path.)

## Say what "no sound" actually means

The most common audio mistake is a negative that means less than you think. "No narration" removes the voiceover — it does **not** silence music or sound effects. If you want genuine silence, say so:

* ❌ `no narration` when you mean a completely silent video — music and SFX can still be added
* ✅ `no audio at all` — the unambiguous way to ask for silence

This mirrors the negatives discipline in [Anatomy](/prompting/anatomy): close the gap explicitly, because the engine acts on the literal words.

## Related

<CardGroup cols={2}>
  <Card title="Vocabulary" href="/prompting/vocabulary">Voice names, caption tones, and audio-reactive mappings</Card>
  <Card title="Captions catalog" href="/prompting/captions-catalog">Styling the timed text this page produces</Card>
  <Card title="Remove background guide" href="/guides/remove-background">The matting command, its person-only caveat, and alternatives</Card>
  <Card title="Video components" href="/guides/video-components">Installable overlays, captions, and effects</Card>
</CardGroup>

<Note>
  **Capstone thread** — the [Level 7 film](/prompting/capstone)'s Material region runs this chapter's entire pipeline on one clip: generated footage → HEVC auto-proxy → background removal mid-scene → word-synced captions from the clip's own transcription, with the clip's audio ducking the BGM (cut from the film, below).
</Note>

This is the clause in the [full capstone prompt](/prompting/capstone#the-full-prompt-verbatim) that buys the piece — prompt language you can lift for your own video:

> \[…] a real talking-head clip (generate a short clip of a person speaking one neutral line via the media pipeline's avatar video generation \[…] **transcode it to HEVC `hvc1`** so the automatic proxy subsystem carries preview) sits as a clip on the wire. The order of operations IS the story: as the camera arrives and BEFORE the person speaks, the framework mattes the footage — **the background peels away via background removal** \[…] THEN they speak, and the main **keywords of their own line — derived from the clip's transcription — land word-synced** \[…] The clip's own audio ducks the BGM briefly; the VO resumes as the camera pulls away.

<DocsVideo title="HyperFrames video: Capstone Region Material" src="https://static.heygen.ai/hyperframes-oss/docs/images/prompting/capstone-region-material.mp4#t=0.1" loop />

*That clause, rendered — the region cut from the finished film.*

*Next: [Design systems and brand](/prompting/design-systems) — pointing the agent at a source of brand truth instead of describing a vibe.*