From 76efbc2d4a99e0d68830976a1f294b6477957bb4 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:09:35 +1000 Subject: [PATCH 01/10] Keep clip timelines continuous after cuts --- src/components/video-editor/clipSplit.test.ts | 19 +++++++++++- src/components/video-editor/clipSplit.ts | 14 +++++++++ .../hooks/useClipRegionCommands.ts | 16 +++++----- src/components/video-editor/types.test.ts | 14 +++++++-- src/components/video-editor/types.ts | 29 +++++++++---------- 5 files changed, 65 insertions(+), 27 deletions(-) diff --git a/src/components/video-editor/clipSplit.test.ts b/src/components/video-editor/clipSplit.test.ts index c9b588c70..9a287e82f 100644 --- a/src/components/video-editor/clipSplit.test.ts +++ b/src/components/video-editor/clipSplit.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from "vitest"; -import { planClipSplit } from "./clipSplit"; +import { planClipSplit, removeSpanAndCloseGap } from "./clipSplit"; import { type ClipRegion, clipsToTrims, getClipSourceEndMs, getClipSourceStartMs, + getTimelineDurationMs, + mapSourceTimeToTimelineTime, mapTimelineTimeToSourceTime, } from "./types"; @@ -127,6 +129,21 @@ describe("planClipSplit", () => { expect(getClipSourceEndMs(kept[1])).toBe(sourceDurationMs); }); + it("closes the timeline gap after deleting the middle of a 3x clip", () => { + const sourceDurationMs = 120_000; + const clip: ClipRegion = { id: "clip-1", startMs: 0, endMs: 40_000, speed: 3 }; + const { kept, deleted } = splitAndDeleteMiddle(clip, 10_000, 10_000); + const closed = removeSpanAndCloseGap([...kept, deleted], deleted); + + expect(closed).toEqual([ + expect.objectContaining({ startMs: 0, endMs: 10_000 }), + expect.objectContaining({ startMs: 10_000, endMs: 30_000, sourceStartMs: 60_000 }), + ]); + expect(mapSourceTimeToTimelineTime(30_000, closed)).toBe(10_000); + expect(mapSourceTimeToTimelineTime(60_000, closed)).toBe(10_000); + expect(getTimelineDurationMs(closed, sourceDurationMs)).toBe(30_000); + }); + it("removes the source range the user cut out at 1x", () => { const sourceDurationMs = 120_000; const clip: ClipRegion = { id: "clip-1", startMs: 0, endMs: sourceDurationMs, speed: 1 }; diff --git a/src/components/video-editor/clipSplit.ts b/src/components/video-editor/clipSplit.ts index 1ff2ebb60..cb250105a 100644 --- a/src/components/video-editor/clipSplit.ts +++ b/src/components/video-editor/clipSplit.ts @@ -6,6 +6,20 @@ export interface ClipSplitPlan { right: ClipRegion; } +export function removeSpanAndCloseGap( + spans: T[], + deleted: { startMs: number; endMs: number }, +): T[] { + const durationMs = deleted.endMs - deleted.startMs; + return spans + .filter((span) => span.endMs <= deleted.startMs || span.startMs >= deleted.endMs) + .map((span) => + span.startMs >= deleted.endMs + ? { ...span, startMs: span.startMs - durationMs, endMs: span.endMs - durationMs } + : span, + ); +} + /** * Split the clip under the playhead into two clips. * diff --git a/src/components/video-editor/hooks/useClipRegionCommands.ts b/src/components/video-editor/hooks/useClipRegionCommands.ts index 40f90cd19..198c4a373 100644 --- a/src/components/video-editor/hooks/useClipRegionCommands.ts +++ b/src/components/video-editor/hooks/useClipRegionCommands.ts @@ -2,7 +2,7 @@ import type { Span } from "dnd-timeline"; import { type Dispatch, type MutableRefObject, type SetStateAction, useCallback } from "react"; import { toast } from "sonner"; import { planClipSpeedChange } from "../clipSpeedChange"; -import { planClipSplit } from "../clipSplit"; +import { planClipSplit, removeSpanAndCloseGap } from "../clipSplit"; import type { AnnotationRegion, AudioRegion, @@ -224,14 +224,14 @@ export function useClipRegionCommands({ const handleClipDelete = useCallback( (id: string) => { const deletedClip = clipRegions.find((clip) => clip.id === id); - setClipRegions((current) => current.filter((clip) => clip.id !== id)); if (deletedClip) { - const outsideDeletedClip = (region: { startMs: number; endMs: number }) => - region.endMs <= deletedClip.startMs || region.startMs >= deletedClip.endMs; - setZoomRegions((current) => current.filter(outsideDeletedClip)); - setAnnotationRegions((current) => current.filter(outsideDeletedClip)); - setSpeedRegions((current) => current.filter(outsideDeletedClip)); - setAudioRegions((current) => current.filter(outsideDeletedClip)); + const closeGap = (regions: T[]) => + removeSpanAndCloseGap(regions, deletedClip); + setClipRegions(closeGap); + setZoomRegions(closeGap); + setAnnotationRegions(closeGap); + setSpeedRegions(closeGap); + setAudioRegions(closeGap); } if (selectedClipId === id) setSelectedClipId(null); }, diff --git a/src/components/video-editor/types.test.ts b/src/components/video-editor/types.test.ts index 4504b8b95..0d95a9da7 100644 --- a/src/components/video-editor/types.test.ts +++ b/src/components/video-editor/types.test.ts @@ -142,6 +142,16 @@ describe("clip timeline mapping", () => { expect(mapSourceTimeToTimelineTime(5_900, clips)).toBe(6_000); }); + it("maps gaps between different source and timeline positions", () => { + const movedClips = [ + { id: "clip-1", startMs: 0, endMs: 4_000, sourceStartMs: 0, speed: 1 }, + { id: "clip-2", startMs: 6_000, endMs: 8_000, sourceStartMs: 10_000, speed: 1 }, + ]; + + expect(mapTimelineTimeToSourceTime(5_900, movedClips)).toBe(10_000); + expect(mapSourceTimeToTimelineTime(9_900, movedClips)).toBe(6_000); + }); + it("finds clips only inside visible kept spans", () => { expect(findClipAtTimelineTime(500, clips)?.id).toBe("clip-1"); expect(findClipAtTimelineTime(5_000, clips)).toBeNull(); @@ -176,10 +186,10 @@ describe("getTimelineDurationMs", () => { ).toBe(20_000); }); - it("keeps the source duration when speed edits make clips shorter", () => { + it("shortens the timeline when speed edits make clips shorter", () => { expect( getTimelineDurationMs([{ id: "clip-1", startMs: 0, endMs: 5_000, speed: 2 }], 10_000), - ).toBe(10_000); + ).toBe(5_000); }); }); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 55de007bb..8f038ca1c 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -259,7 +259,7 @@ export function getTimelineDurationMs(clips: ClipRegion[], sourceDurationMs: num return clips.reduce( (durationMs, clip) => Math.max(durationMs, Math.max(0, Math.round(clip.endMs))), - baseDurationMs, + 0, ); } @@ -271,25 +271,22 @@ function getSafeClipSpeed(clip: ClipRegion) { return Number.isFinite(clip.speed) && clip.speed > 0 ? clip.speed : 1; } -function clampToNearestClipBoundary( - timeMs: number, - clips: ClipRegion[], - kind: "timeline" | "source", -) { +function mapNearestClipBoundary(timeMs: number, clips: ClipRegion[], from: "timeline" | "source") { let nearestTimeMs = Math.round(timeMs); let nearestDistance = Number.POSITIVE_INFINITY; for (const clip of clips) { - const boundaries = - kind === "timeline" - ? [clip.startMs, clip.endMs] - : [getClipSourceStartMs(clip), getClipSourceEndMs(clip)]; - - for (const boundary of boundaries) { - const distance = Math.abs(timeMs - boundary); + const boundaries = [ + [clip.startMs, getClipSourceStartMs(clip)], + [clip.endMs, getClipSourceEndMs(clip)], + ]; + + for (const [timelineTimeMs, sourceTimeMs] of boundaries) { + const inputTimeMs = from === "timeline" ? timelineTimeMs : sourceTimeMs; + const distance = Math.abs(timeMs - inputTimeMs); if (distance < nearestDistance) { nearestDistance = distance; - nearestTimeMs = Math.round(boundary); + nearestTimeMs = Math.round(from === "timeline" ? sourceTimeMs : timelineTimeMs); } } } @@ -315,7 +312,7 @@ export function mapTimelineTimeToSourceTime(timeMs: number, clips: ClipRegion[]) return roundedTimeMs; } - return clampToNearestClipBoundary(roundedTimeMs, sortedClips, "timeline"); + return mapNearestClipBoundary(roundedTimeMs, sortedClips, "timeline"); } export function mapSourceTimeToTimelineTime(timeMs: number, clips: ClipRegion[]): number { @@ -336,7 +333,7 @@ export function mapSourceTimeToTimelineTime(timeMs: number, clips: ClipRegion[]) return roundedTimeMs; } - return clampToNearestClipBoundary(roundedTimeMs, sortedClips, "source"); + return mapNearestClipBoundary(roundedTimeMs, sortedClips, "source"); } export function findClipAtTimelineTime(timeMs: number, clips: ClipRegion[]): ClipRegion | null { From 21f15aa9b71ad14ebad7103fc35ba9c1d158150e Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:25:24 +1000 Subject: [PATCH 02/10] Keep source speed regions anchored during clip deletion --- .../video-editor/hooks/useClipRegionCommands.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/components/video-editor/hooks/useClipRegionCommands.ts b/src/components/video-editor/hooks/useClipRegionCommands.ts index 198c4a373..9963c9ba3 100644 --- a/src/components/video-editor/hooks/useClipRegionCommands.ts +++ b/src/components/video-editor/hooks/useClipRegionCommands.ts @@ -11,7 +11,7 @@ import type { SpeedRegion, ZoomRegion, } from "../types"; -import { getClipSourceStartMs } from "../types"; +import { getClipSourceEndMs, getClipSourceStartMs } from "../types"; type Translator = ( key: string, @@ -230,8 +230,14 @@ export function useClipRegionCommands({ setClipRegions(closeGap); setZoomRegions(closeGap); setAnnotationRegions(closeGap); - setSpeedRegions(closeGap); setAudioRegions(closeGap); + setSpeedRegions((current) => + current.filter( + (region) => + region.endMs <= getClipSourceStartMs(deletedClip) || + region.startMs >= getClipSourceEndMs(deletedClip), + ), + ); } if (selectedClipId === id) setSelectedClipId(null); }, From ad155d486e42d51e4755b524629c79f7719d2805 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Mon, 14 Sep 2026 05:16:16 +1000 Subject: [PATCH 03/10] Preserve clip gaps in preview and export --- src/components/video-editor/VideoPlayback.tsx | 163 +++++++------- .../video-editor/audio/useAudioPreviewSync.ts | 24 +-- .../video-editor/audio/useVideoEditorAudio.ts | 17 +- .../video-editor/clipSpanChange.test.ts | 28 +++ src/components/video-editor/clipSpanChange.ts | 21 ++ src/components/video-editor/clipSplit.test.ts | 16 +- src/components/video-editor/clipSplit.ts | 14 -- .../export/buildExportRenderOptions.ts | 1 + .../hooks/useClipRegionCommands.ts | 98 +-------- .../hooks/useEditorGlobalInteractions.ts | 2 +- .../hooks/useEditorPlaybackControls.ts | 10 +- .../hooks/useTimelineEditingController.ts | 8 +- .../hooks/useTimelineProjection.ts | 15 +- .../layout/EditorPreviewPanel.tsx | 1 - .../layout/EditorVideoPreview.tsx | 7 +- .../layout/useEditorSettingsPanelProps.ts | 7 +- .../project/useProjectLibraryController.ts | 21 +- .../project/useProjectLifecycle.ts | 3 +- src/components/video-editor/types.ts | 5 +- .../videoPlayback/clipPlayback.test.ts | 119 +++++++++++ .../videoPlayback/clipPlayback.ts | 107 ++++++++++ .../video-editor/videoPlayback/index.ts | 1 - .../video-editor/videoPlayback/sceneMotion.ts | 4 +- .../videoPlayback/videoEventHandlers.test.ts | 199 ----------------- .../videoPlayback/videoEventHandlers.ts | 200 ------------------ src/lib/exporter/audioEncoder.ts | 2 + src/lib/exporter/audioProcessorShared.ts | 2 + src/lib/exporter/audioTimelineProcessor.ts | 1 + src/lib/exporter/clipAudioTimeline.test.ts | 105 +++++++++ src/lib/exporter/clipTimeline.ts | 12 ++ src/lib/exporter/frameRenderer.test.ts | 14 ++ src/lib/exporter/frameRenderer.ts | 47 +++- src/lib/exporter/gifExporter.ts | 4 + src/lib/exporter/modernFrameRenderer.test.ts | 23 ++ src/lib/exporter/modernFrameRenderer.ts | 43 +++- src/lib/exporter/modernVideoExporter.ts | 13 +- src/lib/exporter/offlineAudioProcessor.ts | 30 ++- src/lib/exporter/streamingDecodePipeline.ts | 59 +++--- src/lib/exporter/streamingDecoder.test.ts | 59 ++++++ src/lib/exporter/streamingDecoder.ts | 94 +++++--- src/lib/exporter/videoExporter.ts | 10 +- .../exporter/videoTimelineSegments.test.ts | 57 +++++ src/lib/exporter/videoTimelineSegments.ts | 49 ++++- 43 files changed, 968 insertions(+), 747 deletions(-) create mode 100644 src/components/video-editor/clipSpanChange.test.ts create mode 100644 src/components/video-editor/clipSpanChange.ts create mode 100644 src/components/video-editor/videoPlayback/clipPlayback.test.ts create mode 100644 src/components/video-editor/videoPlayback/clipPlayback.ts delete mode 100644 src/components/video-editor/videoPlayback/videoEventHandlers.test.ts delete mode 100644 src/components/video-editor/videoPlayback/videoEventHandlers.ts create mode 100644 src/lib/exporter/clipAudioTimeline.test.ts create mode 100644 src/lib/exporter/clipTimeline.ts create mode 100644 src/lib/exporter/videoTimelineSegments.test.ts diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 87e36a853..3c722e06f 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -77,8 +77,6 @@ import { DEFAULT_ZOOM_OUT_EASING, getDefaultCaptionFontFamily, type Padding, - type SpeedRegion, - type TrimRegion, type WebcamOverlaySettings, type ZoomDepth, type ZoomFocus, @@ -118,7 +116,8 @@ import { resolveSceneZoomTarget, shouldComposePreviewFrame, } from "./videoPlayback/sceneMotion"; -import { createVideoEventHandlers } from "./videoPlayback/videoEventHandlers"; +import { createClipPlayback } from "./videoPlayback/clipPlayback"; +import { type ClipRegion, findClipAtTimelineTime, mapTimelineTimeToSourceTime } from "./types"; import { getWebcamMediaTargetTimeSeconds, isWebcamMediaSynchronized, @@ -232,6 +231,7 @@ function getEffectiveNativeAspectRatio( } interface VideoPlaybackProps { + clipRegions: ClipRegion[]; videoPath: string; onDurationChange: (duration: number) => void; onPreviewReadyChange?: (ready: boolean) => void; @@ -262,8 +262,6 @@ interface VideoPlaybackProps { cropRegion?: import("./types").CropRegion; webcam?: WebcamOverlaySettings; webcamVideoPath?: string | null; - trimRegions?: TrimRegion[]; - speedRegions?: SpeedRegion[]; aspectRatio: AspectRatio; annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; @@ -302,6 +300,8 @@ interface VideoPlaybackProps { } export interface VideoPlaybackRef { + readonly isPlaying: boolean; + seekTimeline: (time: number) => void; video: HTMLVideoElement | null; app: Application | null; videoSprite: Sprite | null; @@ -320,7 +320,8 @@ const VideoPlayback = forwardRef( onDurationChange, onPreviewReadyChange, onTimeUpdate, - currentTime, + currentTime: timelineTime, + clipRegions, onPlayStateChange, onError, wallpaper, @@ -346,8 +347,6 @@ const VideoPlayback = forwardRef( cropRegion, webcam, webcamVideoPath, - trimRegions = [], - speedRegions = [], aspectRatio, annotationRegions = [], autoCaptions = [], @@ -404,7 +403,6 @@ const VideoPlayback = forwardRef( const zoomBlurFilterRef = useRef(null); const motionBlurFilterRef = useRef(null); const cameraContainerRef = useRef(null); - const timeUpdateAnimationRef = useRef(null); const [pixiReady, setPixiReady] = useState(false); const [videoReady, setVideoReady] = useState(false); const [pixiRendererError, setPixiRendererError] = useState(null); @@ -445,7 +443,19 @@ const VideoPlayback = forwardRef( const [captionEditSession, setCaptionEditSession] = useState( null, ); + const currentTime = mapTimelineTimeToSourceTime(timelineTime * 1000, clipRegions) / 1000; + const isGap = !findClipAtTimelineTime(timelineTime * 1000, clipRegions); + const clipRegionsRef = useRef(clipRegions); + const clipPlaybackRef = useRef | null>(null); + const onPlaybackErrorRef = useRef(onError); + onPlaybackErrorRef.current = onError; + const timelineTimeRef = useRef(timelineTime); + timelineTimeRef.current = timelineTime; const currentTimeRef = useRef(0); + useEffect(() => { + clipRegionsRef.current = clipRegions; + clipPlaybackRef.current?.refresh(); + }, [clipRegions]); const zoomRegionsRef = useRef([]); const selectedZoomIdRef = useRef(null); const animationStateRef = useRef(createPlaybackAnimationState()); @@ -474,14 +484,11 @@ const VideoPlayback = forwardRef( const suspendRenderingRef = useRef(suspendRendering); const isSeekingRef = useRef(false); const shouldSnapPausedFrameRef = useRef(false); - const allowPlaybackRef = useRef(false); const lockedVideoDimensionsRef = useRef<{ width: number; height: number; } | null>(null); const layoutVideoContentRef = useRef<(() => void) | null>(null); - const trimRegionsRef = useRef([]); - const speedRegionsRef = useRef([]); const lastWebcamSyncTimeRef = useRef(null); const lastBackgroundSyncTimeRef = useRef(null); const bgVideoRef = useRef(null); @@ -702,15 +709,14 @@ const VideoPlayback = forwardRef( return; } - videoRef.current?.pause(); - onPlayStateChange(false); + clipPlaybackRef.current?.pause(); const nextSession = { target: activeCaptionLayout.editTarget, draft: activeCaptionLayout.editTarget.text, }; captionEditSessionRef.current = nextSession; setCaptionEditSession(nextSession); - }, [activeCaptionLayout, onEditAutoCaption, onPlayStateChange]); + }, [activeCaptionLayout, onEditAutoCaption]); const commitCaptionEdit = useCallback(() => { const session = captionEditSessionRef.current; @@ -1120,29 +1126,20 @@ const VideoPlayback = forwardRef( }, [zoomRegions, selectedZoomId]); useImperativeHandle(ref, () => ({ + get isPlaying() { + return clipPlaybackRef.current?.isPlaying ?? false; + }, + seekTimeline: (time) => clipPlaybackRef.current?.seek(time), video: videoRef.current, app: appRef.current, videoSprite: videoSpriteRef.current, videoContainer: videoContainerRef.current, containerRef, play: async () => { - const vid = videoRef.current; - if (!vid) return; - try { - allowPlaybackRef.current = true; - await vid.play(); - } catch (error) { - allowPlaybackRef.current = false; - throw error; - } + await clipPlaybackRef.current?.play(); }, pause: () => { - const video = videoRef.current; - allowPlaybackRef.current = false; - if (!video) { - return; - } - video.pause(); + clipPlaybackRef.current?.pause(); }, cancelCaptionEdit, refreshFrame: async () => { @@ -1274,12 +1271,12 @@ const VideoPlayback = forwardRef( shouldClearSelectedAnnotation( annotationRegions ?? [], selectedAnnotationId, - Math.round(currentTime * 1000), + Math.round(timelineTime * 1000), ) ) { onSelectAnnotation(null); } - }, [annotationRegions, currentTime, onSelectAnnotation, selectedAnnotationId]); + }, [annotationRegions, timelineTime, onSelectAnnotation, selectedAnnotationId]); useEffect(() => { isPlayingRef.current = isPlaying; @@ -1343,26 +1340,21 @@ const VideoPlayback = forwardRef( } }, [pixiReady, suspendRendering]); - // Keep video wallpapers locked to the same source timestamp as the main clip. + // Backgrounds run on the output timeline, including empty clip intervals. useEffect(() => { const bgVideo = bgVideoRef.current; if (!bgVideo) return; - const clipTimelineTime = currentTime; + const clipTimelineTime = timelineTime; const videoDuration = Number.isFinite(bgVideo.duration) && bgVideo.duration > 0 ? bgVideo.duration : null; const targetTime = videoDuration ? clipTimelineTime % videoDuration : clampMediaTimeToDuration(clipTimelineTime, videoDuration); - const activeSpeedRegion = speedRegionsRef.current.find( - (region) => - currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; enablePitchPreservingPlayback(bgVideo); const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, + basePlaybackRate: 1, currentTime: bgVideo.currentTime, targetTime, toleranceSeconds: 0.02, @@ -1396,15 +1388,7 @@ const VideoPlayback = forwardRef( } lastBackgroundSyncTimeRef.current = clipTimelineTime; - }, [currentTime, isPlaying]); - - useEffect(() => { - trimRegionsRef.current = trimRegions; - }, [trimRegions]); - - useEffect(() => { - speedRegionsRef.current = speedRegions; - }, [speedRegions]); + }, [timelineTime, isPlaying]); useEffect(() => { if (!pixiReady) return; @@ -1729,11 +1713,8 @@ const VideoPlayback = forwardRef( lastWebcamSyncTimeRef.current = targetTime; } - const timelineTimeMs = currentTime * 1000; - const activeSpeedRegion = speedRegionsRef.current.find( - (region) => timelineTimeMs >= region.startMs && timelineTimeMs < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + const targetPlaybackRate = + findClipAtTimelineTime(timelineTime * 1000, clipRegions)?.speed ?? 1; enablePitchPreservingPlayback(webcamVideo); if (Math.abs(webcamVideo.playbackRate - targetPlaybackRate) > 0.001) { webcamVideo.playbackRate = targetPlaybackRate; @@ -1767,7 +1748,14 @@ const VideoPlayback = forwardRef( } lastWebcamSyncTimeRef.current = targetTime; - }, [currentTime, isPlaying, webcamEnabled, webcamTimeOffsetMs, webcamVideoPath]); + }, [ + timelineTime, + clipRegions, + isPlaying, + webcamEnabled, + webcamTimeOffsetMs, + webcamVideoPath, + ]); const handleWebcamMediaReady = useCallback( (event: React.SyntheticEvent) => { @@ -1953,7 +1941,6 @@ const VideoPlayback = forwardRef( if (!video) return; video.pause(); video.currentTime = 0; - allowPlaybackRef.current = false; lastRenderedContentTimeRef.current = null; shouldSnapPausedFrameRef.current = true; lockedVideoDimensionsRef.current = null; @@ -2009,34 +1996,39 @@ const VideoPlayback = forwardRef( layoutVideoContentRef.current?.(); video.pause(); - const { handlePlay, handlePause, handleSeeked, handleSeeking, dispose } = - createVideoEventHandlers({ - video, - isSeekingRef, - shouldSnapPausedFrameRef, - isPlayingRef, - allowPlaybackRef, - currentTimeRef, - timeUpdateAnimationRef, - onPlayStateChange, - onTimeUpdate, - trimRegionsRef, - speedRegionsRef, - }); - - video.addEventListener("play", handlePlay); - video.addEventListener("pause", handlePause); - video.addEventListener("ended", handlePause); + const transport = createClipPlayback({ + video, + getClips: () => clipRegionsRef.current, + onTime: (time, source) => { + if (source !== null) currentTimeRef.current = source * 1000; + onTimeUpdate(time); + }, + onPlaying: (playing) => { + isPlayingRef.current = playing; + onPlayStateChange(playing); + }, + onError: (error) => + onPlaybackErrorRef.current( + error instanceof Error ? error.message : String(error), + ), + }); + clipPlaybackRef.current = transport; + transport.seek(timelineTimeRef.current); + const handleSeeked = () => { + isSeekingRef.current = false; + }; + const handleSeeking = () => { + isSeekingRef.current = true; + shouldSnapPausedFrameRef.current = true; + }; video.addEventListener("seeked", handleSeeked); video.addEventListener("seeking", handleSeeking); return () => { - video.removeEventListener("play", handlePlay); - video.removeEventListener("pause", handlePause); - video.removeEventListener("ended", handlePause); video.removeEventListener("seeked", handleSeeked); video.removeEventListener("seeking", handleSeeking); - dispose(); + transport.dispose(); + clipPlaybackRef.current = null; videoEffectsContainer.mask = null; videoContainer.mask = null; @@ -2083,7 +2075,7 @@ const VideoPlayback = forwardRef( motionBlurTuning: zoomMotionBlurTuningRef.current, transformOverride: transform, motionBlurState: motionBlurStateRef.current, - frameTimeMs: currentTimeRef.current, + frameTimeMs: timelineTimeRef.current * 1000, }); state.x = appliedTransform.x; @@ -2114,7 +2106,7 @@ const VideoPlayback = forwardRef( // The export compositor advances exactly once for each output timestamp. // Do the same here: repeated Pixi ticks at one media timestamp must not // advance cursor springs or clear the blur calculated for that frame. - const contentTimeMs = currentTimeRef.current; + const contentTimeMs = timelineTimeRef.current * 1000; const previousContentTimeMs = lastRenderedContentTimeRef.current; const deltaMs = previousContentTimeMs !== null @@ -2141,7 +2133,8 @@ const VideoPlayback = forwardRef( const target = resolveSceneZoomTarget({ zoomRegions: zoomRegionsRef.current, - timeMs: currentTimeRef.current, + timeMs: timelineTimeRef.current * 1000, + cursorTimeMs: currentTimeRef.current, connectZooms: connectZoomsRef.current, zoomInDurationMs: zoomInDurationMsRef.current, zoomOutDurationMs: zoomOutDurationMsRef.current, @@ -2316,7 +2309,6 @@ const VideoPlayback = forwardRef( ); video.currentTime = targetTime; video.pause(); - allowPlaybackRef.current = false; currentTimeRef.current = targetTime * 1000; if (videoReadyRafRef.current) { @@ -2524,6 +2516,7 @@ const VideoPlayback = forwardRef( className="absolute inset-0" style={{ filter: sceneEffects.shadowFilter, + visibility: isGap ? "hidden" : "visible", }} /> {hasRendererFallback && ( @@ -2557,7 +2550,7 @@ const VideoPlayback = forwardRef( ref={webcamBubbleRef} className="absolute" style={{ - display: webcam.enabled ? "block" : "none", + display: webcam.enabled && !isGap ? "block" : "none", pointerEvents: "none", }} > @@ -2597,7 +2590,7 @@ const VideoPlayback = forwardRef( ) : null} - {activeCaptionLayout && autoCaptionSettings ? ( + {!isGap && activeCaptionLayout && autoCaptionSettings ? (
( }} > {(() => { - const timeMs = Math.round(currentTime * 1000); + const timeMs = Math.round(timelineTime * 1000); const filtered = (annotationRegions || []).filter( (annotation) => isAnnotationActiveAtTime(annotation, timeMs), diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index e7a5c8a5a..c1e3a2594 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -8,7 +8,7 @@ import { getMediaSyncPlaybackRate, resolvePreviewMediaDuration, } from "@/lib/mediaTiming"; -import type { AudioRegion, SpeedRegion } from "../types"; +import type { AudioRegion } from "../types"; import { getAudioResourceVersionKey, getVersionedAudioResourceUrl, @@ -25,7 +25,7 @@ interface UseAudioPreviewSyncParams { currentTime: number; timelineTime: number; duration: number; - effectiveSpeedRegions: SpeedRegion[]; + sourcePlaybackRate: number; previewSourceAudioFallbackPaths: string[]; sourceAudioFallbackStartDelayMsByPath: Record; sourceAudioResourceVersion: number; @@ -41,7 +41,7 @@ export function useAudioPreviewSync({ currentTime, timelineTime, duration, - effectiveSpeedRegions, + sourcePlaybackRate, previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioResourceVersion, @@ -344,10 +344,6 @@ export function useAudioPreviewSync({ useEffect(() => { const currentTimeMs = timelineTime * 1000; - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; for (const track of resolvedUserTracks) { const audio = audioElementsRef.current.get(track.id); @@ -364,7 +360,7 @@ export function useAudioPreviewSync({ audio.currentTime = audioOffset; } const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, + basePlaybackRate: 1, currentTime: audio.currentTime, targetTime: audioOffset, }); @@ -378,7 +374,7 @@ export function useAudioPreviewSync({ audio.pause(); } } - }, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]); + }, [isPlaying, resolvedUserTracks, timelineTime]); useEffect(() => { if (resolvedSourceTracks.length === 0) { @@ -386,10 +382,6 @@ export function useAudioPreviewSync({ return; } - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; const previousTimelineTime = lastSourceAudioSyncTimeRef.current; const timelineJumped = previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; @@ -450,13 +442,13 @@ export function useAudioPreviewSync({ // KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter // from continuous micro-corrections on system audio. - const syncedPlaybackRate = targetPlaybackRate; + const syncedPlaybackRate = sourcePlaybackRate; if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { audio.playbackRate = syncedPlaybackRate; } const atEnd = audioDuration !== null && targetTime >= audioDuration; - if (isPlaying && !beforeAudioStart && !atEnd) { + if (isPlaying && !isCurrentClipMuted && !beforeAudioStart && !atEnd) { void ensureSourceAudioRunning().then(() => { audio.play().catch(() => undefined); }); @@ -469,7 +461,7 @@ export function useAudioPreviewSync({ }, [ currentTime, duration, - effectiveSpeedRegions, + sourcePlaybackRate, getSourceTrackPreviewGain, isCurrentClipMuted, isPlaying, diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index f56304340..4d14131d4 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -1,8 +1,9 @@ import React, { useMemo } from "react"; import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; -import type { AudioRegion, ClipRegion, SpeedRegion } from "../types"; -import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio"; +import type { AudioRegion, ClipRegion } from "../types"; +import { findClipAtTimelineTime } from "../types"; +import { isClipMutedById } from "./clipAudio"; import { useAudioPreviewSync } from "./useAudioPreviewSync"; import { useClipAudioSettingsController } from "./useClipAudioSettingsController"; import { useSourceAudioFallback } from "./useSourceAudioFallback"; @@ -27,7 +28,6 @@ interface UseVideoEditorAudioParams { selectedClipId: string | null; clipRegions: ClipRegion[]; audioRegions: AudioRegion[]; - effectiveSpeedRegions: SpeedRegion[]; sourceAudioTrackSettingsByClip: Record; setSourceAudioTrackSettingsByClip: React.Dispatch< React.SetStateAction> @@ -51,7 +51,6 @@ export function useVideoEditorAudio({ selectedClipId, clipRegions, audioRegions, - effectiveSpeedRegions, sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, @@ -85,11 +84,13 @@ export function useVideoEditorAudio({ const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview; const activeClipIdAtCurrentTime = useMemo( - () => getActiveClipIdAtSourceTime(currentTime, clipRegions), - [clipRegions, currentTime], + () => findClipAtTimelineTime(timelineTime * 1000, clipRegions)?.id ?? null, + [clipRegions, timelineTime], ); const isCurrentClipMuted = useMemo( - () => isClipMutedById(activeClipIdAtCurrentTime, clipRegions), + () => + activeClipIdAtCurrentTime === null || + isClipMutedById(activeClipIdAtCurrentTime, clipRegions), [activeClipIdAtCurrentTime, clipRegions], ); @@ -119,7 +120,7 @@ export function useVideoEditorAudio({ currentTime, timelineTime, duration, - effectiveSpeedRegions, + sourcePlaybackRate: findClipAtTimelineTime(timelineTime * 1000, clipRegions)?.speed ?? 1, previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioResourceVersion: sourceAudioFallbackRefreshKey, diff --git a/src/components/video-editor/clipSpanChange.test.ts b/src/components/video-editor/clipSpanChange.test.ts new file mode 100644 index 000000000..abc7be35f --- /dev/null +++ b/src/components/video-editor/clipSpanChange.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { changeClipSpan } from "./clipSpanChange"; + +describe("clip span changes", () => { + const clip = { id: "clip", startMs: 1000, endMs: 4000, sourceStartMs: 3000, speed: 3 }; + it("moves footage without changing its source window", () => { + expect(changeClipSpan(clip, 2000, 5000, 12000)).toEqual({ + ...clip, + startMs: 2000, + endMs: 5000, + }); + }); + it("advances the source in-point by the trimmed duration times speed", () => { + expect(changeClipSpan(clip, 2000, 4000, 12000)).toEqual({ + ...clip, + startMs: 2000, + sourceStartMs: 6000, + }); + }); + it("does not extend beyond source EOF after a move", () => { + const moved = changeClipSpan(clip, 2000, 5000, 12000); + expect(changeClipSpan(moved, 2000, 6000, 12000)).toEqual(moved); + }); + it("does not extend a moved clip before its source starts", () => { + const moved = { ...clip, startMs: 2000, endMs: 6000, sourceStartMs: 0 }; + expect(changeClipSpan(moved, 1000, 6000, 12000)).toEqual(moved); + }); +}); diff --git a/src/components/video-editor/clipSpanChange.ts b/src/components/video-editor/clipSpanChange.ts new file mode 100644 index 000000000..985a50be2 --- /dev/null +++ b/src/components/video-editor/clipSpanChange.ts @@ -0,0 +1,21 @@ +import { type ClipRegion, getClipSourceStartMs } from "./types"; + +export function changeClipSpan( + clip: ClipRegion, + startMs: number, + endMs: number, + sourceDurationMs: number, +): ClipRegion { + const sourceStart = getClipSourceStartMs(clip); + const isMove = startMs - clip.startMs === endMs - clip.endMs; + if (isMove) return { ...clip, startMs, endMs, sourceStartMs: sourceStart }; + + // Resizing reveals/hides footage; it cannot manufacture source before 0 or after EOF. + const start = Math.max(startMs, Math.ceil(clip.startMs - sourceStart / clip.speed)); + const sourceStartMs = Math.round(sourceStart + (start - clip.startMs) * clip.speed); + const end = Math.min( + endMs, + Math.floor(start + (sourceDurationMs - sourceStartMs) / clip.speed), + ); + return { ...clip, startMs: start, endMs: end, sourceStartMs }; +} diff --git a/src/components/video-editor/clipSplit.test.ts b/src/components/video-editor/clipSplit.test.ts index 9a287e82f..c50741f5c 100644 --- a/src/components/video-editor/clipSplit.test.ts +++ b/src/components/video-editor/clipSplit.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { planClipSplit, removeSpanAndCloseGap } from "./clipSplit"; +import { planClipSplit } from "./clipSplit"; import { type ClipRegion, clipsToTrims, @@ -129,19 +129,19 @@ describe("planClipSplit", () => { expect(getClipSourceEndMs(kept[1])).toBe(sourceDurationMs); }); - it("closes the timeline gap after deleting the middle of a 3x clip", () => { + it("preserves the gap and source positions after deleting the middle of a 3x clip", () => { const sourceDurationMs = 120_000; const clip: ClipRegion = { id: "clip-1", startMs: 0, endMs: 40_000, speed: 3 }; const { kept, deleted } = splitAndDeleteMiddle(clip, 10_000, 10_000); - const closed = removeSpanAndCloseGap([...kept, deleted], deleted); + const remaining = [...kept, deleted].filter(({ id }) => id !== deleted.id); - expect(closed).toEqual([ + expect(remaining).toEqual([ expect.objectContaining({ startMs: 0, endMs: 10_000 }), - expect.objectContaining({ startMs: 10_000, endMs: 30_000, sourceStartMs: 60_000 }), + expect.objectContaining({ startMs: 20_000, endMs: 40_000, sourceStartMs: 60_000 }), ]); - expect(mapSourceTimeToTimelineTime(30_000, closed)).toBe(10_000); - expect(mapSourceTimeToTimelineTime(60_000, closed)).toBe(10_000); - expect(getTimelineDurationMs(closed, sourceDurationMs)).toBe(30_000); + expect(mapSourceTimeToTimelineTime(30_000, remaining)).toBe(10_000); + expect(mapSourceTimeToTimelineTime(60_000, remaining)).toBe(20_000); + expect(getTimelineDurationMs(remaining, sourceDurationMs)).toBe(40_000); }); it("removes the source range the user cut out at 1x", () => { diff --git a/src/components/video-editor/clipSplit.ts b/src/components/video-editor/clipSplit.ts index cb250105a..1ff2ebb60 100644 --- a/src/components/video-editor/clipSplit.ts +++ b/src/components/video-editor/clipSplit.ts @@ -6,20 +6,6 @@ export interface ClipSplitPlan { right: ClipRegion; } -export function removeSpanAndCloseGap( - spans: T[], - deleted: { startMs: number; endMs: number }, -): T[] { - const durationMs = deleted.endMs - deleted.startMs; - return spans - .filter((span) => span.endMs <= deleted.startMs || span.startMs >= deleted.endMs) - .map((span) => - span.startMs >= deleted.endMs - ? { ...span, startMs: span.startMs - durationMs, endMs: span.endMs - durationMs } - : span, - ); -} - /** * Split the clip under the playhead into two clips. * diff --git a/src/components/video-editor/export/buildExportRenderOptions.ts b/src/components/video-editor/export/buildExportRenderOptions.ts index 4ec4214dd..0f240e368 100644 --- a/src/components/video-editor/export/buildExportRenderOptions.ts +++ b/src/components/video-editor/export/buildExportRenderOptions.ts @@ -33,6 +33,7 @@ export function buildExportRenderOptions({ onProgress, }: BuildExportRenderOptionsInput) { return { + clipRegions: timeline.clipRegions, wallpaper: appearance.wallpaper, trimRegions: timeline.trimRegions, speedRegions: effectiveSpeedRegions, diff --git a/src/components/video-editor/hooks/useClipRegionCommands.ts b/src/components/video-editor/hooks/useClipRegionCommands.ts index 9963c9ba3..3470aa07b 100644 --- a/src/components/video-editor/hooks/useClipRegionCommands.ts +++ b/src/components/video-editor/hooks/useClipRegionCommands.ts @@ -2,16 +2,9 @@ import type { Span } from "dnd-timeline"; import { type Dispatch, type MutableRefObject, type SetStateAction, useCallback } from "react"; import { toast } from "sonner"; import { planClipSpeedChange } from "../clipSpeedChange"; -import { planClipSplit, removeSpanAndCloseGap } from "../clipSplit"; -import type { - AnnotationRegion, - AudioRegion, - ClipRegion, - EditorEffectSection, - SpeedRegion, - ZoomRegion, -} from "../types"; -import { getClipSourceEndMs, getClipSourceStartMs } from "../types"; +import { changeClipSpan } from "../clipSpanChange"; +import { planClipSplit } from "../clipSplit"; +import type { ClipRegion, EditorEffectSection, ZoomRegion } from "../types"; type Translator = ( key: string, @@ -20,13 +13,11 @@ type Translator = ( ) => string; interface UseClipRegionCommandsParams { + sourceDurationMs: number; clipRegions: ClipRegion[]; setClipRegions: Dispatch>; zoomRegions: ZoomRegion[]; setZoomRegions: Dispatch>; - setAnnotationRegions: Dispatch>; - setSpeedRegions: Dispatch>; - setAudioRegions: Dispatch>; selectedClipId: string | null; setSelectedClipId: Dispatch>; setSelectedZoomId: Dispatch>; @@ -39,13 +30,11 @@ interface UseClipRegionCommandsParams { } export function useClipRegionCommands({ + sourceDurationMs, clipRegions, setClipRegions, zoomRegions, setZoomRegions, - setAnnotationRegions, - setSpeedRegions, - setAudioRegions, selectedClipId, setSelectedClipId, setSelectedZoomId, @@ -102,16 +91,6 @@ export function useClipRegionCommands({ const oldClip = clipRegions.find((clip) => clip.id === id); const newStart = Math.round(span.start); const newEnd = Math.round(span.end); - const removedSegments = oldClip - ? [ - ...(newStart > oldClip.startMs - ? [{ startMs: oldClip.startMs, endMs: newStart }] - : []), - ...(newEnd < oldClip.endMs - ? [{ startMs: newEnd, endMs: oldClip.endMs }] - : []), - ] - : []; if (oldClip) { const startDelta = newStart - oldClip.startMs; @@ -131,48 +110,14 @@ export function useClipRegionCommands({ } } - if (removedSegments.length > 0) { - const removeTrimmedRegions = ( - regions: T[], - ) => - regions.filter( - (region) => - !removedSegments.some( - (segment) => - region.startMs < segment.endMs && - region.endMs > segment.startMs, - ), - ); - setZoomRegions((current) => removeTrimmedRegions(current)); - setAnnotationRegions((current) => removeTrimmedRegions(current)); - setSpeedRegions((current) => removeTrimmedRegions(current)); - setAudioRegions((current) => removeTrimmedRegions(current)); - } - setClipRegions((current) => current.map((clip) => { if (clip.id !== id) return clip; - const speed = Number.isFinite(clip.speed) && clip.speed > 0 ? clip.speed : 1; - const startDelta = newStart - clip.startMs; - const endDelta = newEnd - clip.endMs; - // A move carries its footage along; trimming the left edge skips - // into the source by however much source that edge covered. - const isMove = Math.abs(startDelta - endDelta) < 1; - const sourceStartMs = isMove - ? getClipSourceStartMs(clip) - : Math.max(0, Math.round(getClipSourceStartMs(clip) + startDelta * speed)); - return { ...clip, startMs: newStart, endMs: newEnd, sourceStartMs }; + return changeClipSpan(clip, newStart, newEnd, sourceDurationMs); }), ); }, - [ - clipRegions, - setAnnotationRegions, - setAudioRegions, - setClipRegions, - setSpeedRegions, - setZoomRegions, - ], + [clipRegions, setClipRegions, setZoomRegions, sourceDurationMs], ); const handleClipSpeedChange = useCallback( @@ -223,34 +168,11 @@ export function useClipRegionCommands({ const handleClipDelete = useCallback( (id: string) => { - const deletedClip = clipRegions.find((clip) => clip.id === id); - if (deletedClip) { - const closeGap = (regions: T[]) => - removeSpanAndCloseGap(regions, deletedClip); - setClipRegions(closeGap); - setZoomRegions(closeGap); - setAnnotationRegions(closeGap); - setAudioRegions(closeGap); - setSpeedRegions((current) => - current.filter( - (region) => - region.endMs <= getClipSourceStartMs(deletedClip) || - region.startMs >= getClipSourceEndMs(deletedClip), - ), - ); - } + // Other tracks have their own timeline positions; deleting footage is not a ripple edit. + setClipRegions((current) => current.filter((clip) => clip.id !== id)); if (selectedClipId === id) setSelectedClipId(null); }, - [ - clipRegions, - selectedClipId, - setAnnotationRegions, - setAudioRegions, - setClipRegions, - setSelectedClipId, - setSpeedRegions, - setZoomRegions, - ], + [selectedClipId, setClipRegions, setSelectedClipId], ); return { diff --git a/src/components/video-editor/hooks/useEditorGlobalInteractions.ts b/src/components/video-editor/hooks/useEditorGlobalInteractions.ts index f5e7ec491..4f47427fa 100644 --- a/src/components/video-editor/hooks/useEditorGlobalInteractions.ts +++ b/src/components/video-editor/hooks/useEditorGlobalInteractions.ts @@ -52,7 +52,7 @@ export function useEditorGlobalInteractions({ event.preventDefault(); const playback = videoPlaybackRef.current; if (!playback?.video) return; - if (playback.video.paused) startPlayback(); + if (!playback.isPlaying) startPlayback(); else playback.pause(); }; window.addEventListener("keydown", handleKeyDown, { capture: true }); diff --git a/src/components/video-editor/hooks/useEditorPlaybackControls.ts b/src/components/video-editor/hooks/useEditorPlaybackControls.ts index 733dee939..dfd28c20c 100644 --- a/src/components/video-editor/hooks/useEditorPlaybackControls.ts +++ b/src/components/video-editor/hooks/useEditorPlaybackControls.ts @@ -6,7 +6,6 @@ interface UseEditorPlaybackControlsParams { videoPlaybackRef: RefObject; timelineRef: RefObject; playSourceAudioPreview: () => void; - mapTimelineTimeToSourceTime: (timeMs: number) => number; timelinePlayheadTime: number; timelineDuration: number; } @@ -15,7 +14,6 @@ export function useEditorPlaybackControls({ videoPlaybackRef, timelineRef, playSourceAudioPreview, - mapTimelineTimeToSourceTime, timelinePlayheadTime, timelineDuration, }: UseEditorPlaybackControlsParams) { @@ -34,7 +32,7 @@ export function useEditorPlaybackControls({ const video = playback?.video; if (!playback || !video) return; - if (!video.paused && !video.ended) playback.pause(); + if (playback.isPlaying) playback.pause(); else startPlayback(); }, [getActivePlayback, startPlayback]); @@ -44,10 +42,10 @@ export function useEditorPlaybackControls({ const video = playback?.video; if (!video) return; - if (options.pause && !video.paused) playback?.pause(); - video.currentTime = mapTimelineTimeToSourceTime(time * 1000) / 1000; + if (options.pause) playback.pause(); + playback.seekTimeline(time); }, - [getActivePlayback, mapTimelineTimeToSourceTime], + [getActivePlayback], ); const handleTimelineSeek = useCallback( diff --git a/src/components/video-editor/hooks/useTimelineEditingController.ts b/src/components/video-editor/hooks/useTimelineEditingController.ts index 4ea311537..cc42f864e 100644 --- a/src/components/video-editor/hooks/useTimelineEditingController.ts +++ b/src/components/video-editor/hooks/useTimelineEditingController.ts @@ -88,13 +88,12 @@ export function useTimelineEditingController(input: Input) { selectedClipId: timeline.selectedClipId, clipRegions: timeline.clipRegions, audioRegions: timeline.audioRegions, - effectiveSpeedRegions: projection.effectiveSpeedRegions, sourceAudioTrackSettingsByClip: timeline.sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip: timeline.setSourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings: timeline.defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings: timeline.setDefaultSourceAudioTrackSettings, - currentTime: input.currentTime, timelineTime: projection.timelinePlayheadTime, + currentTime: projection.mapTimelineTimeToSourceTime(input.currentTime * 1000) / 1000, duration: input.duration, isPlaying: input.isPlaying, previewVolume: input.previewVolume, @@ -106,7 +105,6 @@ export function useTimelineEditingController(input: Input) { videoPlaybackRef: input.videoPlaybackRef, timelineRef: input.timelineRef, playSourceAudioPreview: audio.playSourceAudioPreview, - mapTimelineTimeToSourceTime: projection.mapTimelineTimeToSourceTime, timelinePlayheadTime: projection.timelinePlayheadTime, timelineDuration: projection.timelineDuration, }); @@ -174,13 +172,11 @@ export function useTimelineEditingController(input: Input) { input.pendingFreshRecordingAutoSuggestTelemetryCountRef, }); const clipCommands = useClipRegionCommands({ + sourceDurationMs: input.duration * 1000, clipRegions: timeline.clipRegions, setClipRegions: timeline.setClipRegions, zoomRegions: timeline.zoomRegions, setZoomRegions: timeline.setZoomRegions, - setAnnotationRegions: timeline.setAnnotationRegions, - setSpeedRegions: timeline.setSpeedRegions, - setAudioRegions: timeline.setAudioRegions, selectedClipId: timeline.selectedClipId, setSelectedClipId: timeline.setSelectedClipId, setSelectedZoomId: timeline.setSelectedZoomId, diff --git a/src/components/video-editor/hooks/useTimelineProjection.ts b/src/components/video-editor/hooks/useTimelineProjection.ts index ea6689682..a455c29ea 100644 --- a/src/components/video-editor/hooks/useTimelineProjection.ts +++ b/src/components/video-editor/hooks/useTimelineProjection.ts @@ -89,15 +89,7 @@ export function useTimelineProjection({ (timeMs: number) => mapSourceTimeToTimelineTime(timeMs, clipRegions), [clipRegions], ); - const effectiveZoomRegions = useMemo( - () => - zoomRegions.map((region) => ({ - ...region, - startMs: toSourceTime(region.startMs), - endMs: toSourceTime(region.endMs), - })), - [zoomRegions, toSourceTime], - ); + const effectiveZoomRegions: ZoomRegion[] = zoomRegions; const effectiveCaptionRegions = useMemo( () => autoCaptions.map((cue) => ({ @@ -107,10 +99,7 @@ export function useTimelineProjection({ })), [autoCaptions, toTimelineTime], ); - const timelinePlayheadTime = useMemo( - () => toTimelineTime(currentTime * 1000) / 1000, - [currentTime, toTimelineTime], - ); + const timelinePlayheadTime = currentTime; const timelineDuration = useMemo( () => getTimelineDurationMs(clipRegions, duration * 1000) / 1000, [clipRegions, duration], diff --git a/src/components/video-editor/layout/EditorPreviewPanel.tsx b/src/components/video-editor/layout/EditorPreviewPanel.tsx index 27fabb64a..1c7ca4869 100644 --- a/src/components/video-editor/layout/EditorPreviewPanel.tsx +++ b/src/components/video-editor/layout/EditorPreviewPanel.tsx @@ -191,7 +191,6 @@ export function EditorPreviewPanel(props: Props) { timeline={timeline} audio={audio} effectiveZoomRegions={projection.effectiveZoomRegions} - effectiveSpeedRegions={projection.effectiveSpeedRegions} effectiveCursorTelemetry={effectiveCursorTelemetry} effectiveShowCursor={effectiveShowCursor} setDuration={setDuration} diff --git a/src/components/video-editor/layout/EditorVideoPreview.tsx b/src/components/video-editor/layout/EditorVideoPreview.tsx index caa665624..ae0fe3e96 100644 --- a/src/components/video-editor/layout/EditorVideoPreview.tsx +++ b/src/components/video-editor/layout/EditorVideoPreview.tsx @@ -3,7 +3,7 @@ import type { AspectRatio } from "@/utils/aspectRatioUtils"; import type { useVideoEditorAudio } from "../audio/useVideoEditorAudio"; import type { useAppearanceState } from "../state/useAppearanceState"; import type { useTimelineState } from "../state/useTimelineState"; -import type { CursorTelemetryPoint, SpeedRegion, ZoomRegion } from "../types"; +import type { CursorTelemetryPoint, ZoomRegion } from "../types"; import VideoPlayback, { type VideoPlaybackRef } from "../VideoPlayback"; type PlaybackProps = ComponentProps; @@ -30,7 +30,6 @@ type Props = { timeline: ReturnType; audio: ReturnType; effectiveZoomRegions: ZoomRegion[]; - effectiveSpeedRegions: SpeedRegion[]; effectiveCursorTelemetry: CursorTelemetryPoint[]; effectiveShowCursor: boolean; setDuration: Dispatch>; @@ -54,7 +53,6 @@ export function EditorVideoPreview({ timeline, audio, effectiveZoomRegions, - effectiveSpeedRegions, effectiveCursorTelemetry, effectiveShowCursor, setDuration, @@ -66,6 +64,7 @@ export function EditorVideoPreview({ }: Props) { return ( 0; + // An explicit empty clip list means the user deleted all footage, not a legacy project. + refs.clipInitializedRef.current = Array.isArray(persistedEditor.clipRegions); refs.autoFullTrackClipIdRef.current = null; refs.autoFullTrackClipEndMsRef.current = null; timeline.setSpeedRegions(editor.speedRegions); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 8f038ca1c..bfa3cb147 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -337,11 +337,8 @@ export function mapSourceTimeToTimelineTime(timeMs: number, clips: ClipRegion[]) } export function findClipAtTimelineTime(timeMs: number, clips: ClipRegion[]): ClipRegion | null { - const roundedTimeMs = Math.round(timeMs); return ( - sortClipRegions(clips).find( - (clip) => roundedTimeMs >= clip.startMs && roundedTimeMs < clip.endMs, - ) ?? null + sortClipRegions(clips).find((clip) => timeMs >= clip.startMs && timeMs < clip.endMs) ?? null ); } diff --git a/src/components/video-editor/videoPlayback/clipPlayback.test.ts b/src/components/video-editor/videoPlayback/clipPlayback.test.ts new file mode 100644 index 000000000..99f35a4ba --- /dev/null +++ b/src/components/video-editor/videoPlayback/clipPlayback.test.ts @@ -0,0 +1,119 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ClipRegion } from "../types"; +import { createClipPlayback } from "./clipPlayback"; + +describe("clip timeline playback", () => { + let now = 0; + let tick: FrameRequestCallback | undefined; + beforeEach(() => { + now = 0; + tick = undefined; + vi.spyOn(performance, "now").mockImplementation(() => now); + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + tick = callback; + return 1; + }); + vi.stubGlobal("cancelAnimationFrame", () => { + tick = undefined; + }); + }); + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + const advance = (milliseconds: number) => { + now += milliseconds; + const callback = tick; + tick = undefined; + callback?.(now); + }; + function setup( + clips: ClipRegion[] = [ + { id: "left", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 3 }, + { id: "right", startMs: 2000, endMs: 4000, sourceStartMs: 6000, speed: 3 }, + ], + ) { + const video = { + duration: 12, + currentTime: 0, + seeking: false, + playbackRate: 1, + play: vi.fn(async () => {}), + pause: vi.fn(), + } as unknown as HTMLVideoElement; + const onTime = vi.fn(); + const onPlaying = vi.fn(); + const onError = vi.fn(); + const playback = createClipPlayback({ + video, + getClips: () => clips, + onTime, + onPlaying, + onError, + }); + return { video, playback, onTime, onPlaying, onError }; + } + it("takes real time through a deleted middle at 3x, then resumes at the retained source in-point", async () => { + const { video, playback, onTime } = setup(); + await playback.play(); + expect(video.playbackRate).toBe(3); + video.currentTime = 3; + advance(1000); + expect(onTime).toHaveBeenLastCalledWith(1, null); + expect(playback.isPlaying).toBe(true); + advance(500); + expect(onTime).toHaveBeenLastCalledWith(1.5, null); + advance(500); + expect(video.currentTime).toBe(6); + expect(onTime).toHaveBeenLastCalledWith(2, 6); + video.currentTime = 9; + advance(1000); + expect(onTime).toHaveBeenLastCalledWith(3, 9); + video.currentTime = 12; + advance(1000); + expect(playback.isPlaying).toBe(false); + }); + it("seeks, pauses and resumes inside a gap without revealing source footage or restarting the gap", async () => { + const { video, playback, onTime } = setup(); + playback.seek(1.25); + expect(onTime).toHaveBeenLastCalledWith(1.25, null); + await playback.play(); + expect(video.play).not.toHaveBeenCalled(); + advance(250); + playback.pause(); + advance(5000); + expect(onTime).toHaveBeenLastCalledWith(1.5, null); + await playback.play(); + advance(250); + expect(onTime).toHaveBeenLastCalledWith(1.75, null); + }); + it("does not skip footage when source playback stalls or a seek is pending", async () => { + const { video, playback, onTime } = setup(); + await playback.play(); + advance(5000); + expect(onTime).toHaveBeenLastCalledWith(0, 0); + playback.seek(3); + Object.assign(video, { seeking: true, currentTime: 0 }); + advance(2000); + expect(onTime).toHaveBeenLastCalledWith(3, 9); + }); + it("plays leading gaps and clips placed earlier than their source positions", async () => { + const { video, playback, onTime } = setup([ + { id: "moved", startMs: 1000, endMs: 2000, sourceStartMs: 9000, speed: 1 }, + ]); + await playback.play(); + advance(500); + expect(onTime).toHaveBeenLastCalledWith(0.5, null); + advance(500); + expect(video.currentTime).toBe(9); + expect(video.playbackRate).toBe(1); + }); + it("stops and reports an actual source playback failure", async () => { + const { video, playback, onError } = setup(); + const error = new DOMException("unsupported", "NotSupportedError"); + vi.mocked(video.play).mockRejectedValue(error); + await playback.play(); + expect(onError).toHaveBeenCalledWith(error); + expect(playback.isPlaying).toBe(false); + }); +}); diff --git a/src/components/video-editor/videoPlayback/clipPlayback.ts b/src/components/video-editor/videoPlayback/clipPlayback.ts new file mode 100644 index 000000000..bf804ea85 --- /dev/null +++ b/src/components/video-editor/videoPlayback/clipPlayback.ts @@ -0,0 +1,107 @@ +import { enablePitchPreservingPlayback } from "@/lib/mediaTiming"; +import { + type ClipRegion, + findClipAtTimelineTime, + getClipSourceStartMs, + getTimelineDurationMs, +} from "../types"; + +/** Timeline time advances at 1x; only the source media uses the clip's speed. */ +export function createClipPlayback({ + video, + getClips, + onTime, + onPlaying, + onError, +}: { + video: HTMLVideoElement; + getClips: () => ClipRegion[]; + onTime: (timelineSeconds: number, sourceSeconds: number | null) => void; + onPlaying: (playing: boolean) => void; + onError: (error: unknown) => void; +}) { + let timeMs = 0; + let playing = false; + let request: number | null = null; + let lastTick = 0; + let activeClip: ClipRegion | null = null; + let playRequest = 0; + const duration = () => getTimelineDurationMs(getClips(), video.duration * 1000); + + const pause = () => { + playRequest++; + playing = false; + if (request !== null) cancelAnimationFrame(request); + request = null; + video.pause(); + onPlaying(false); + }; + const playSource = () => { + const request = ++playRequest; + void video.play().catch((error) => { + // A deliberate seek/pause can interrupt a pending play request. + if (!playing || request !== playRequest) return; + pause(); + onError(error); + }); + }; + const sync = (seek = false) => { + const clip = findClipAtTimelineTime(timeMs, getClips()); + const sourceMs = clip + ? getClipSourceStartMs(clip) + (timeMs - clip.startMs) * clip.speed + : null; + if (clip && sourceMs !== null) { + enablePitchPreservingPlayback(video); + video.playbackRate = clip.speed; + if (seek || clip !== activeClip) video.currentTime = sourceMs / 1000; + if (playing && (seek || clip !== activeClip)) playSource(); + } else { + playRequest++; + video.pause(); + } + activeClip = clip; + onTime(timeMs / 1000, sourceMs === null ? null : sourceMs / 1000); + }; + const tick = (now: number) => { + request = null; + if (!playing) return; + // Follow the media inside footage (buffering must not skip content). + // A gap has no source clock, so advance it with elapsed real time. + if (!activeClip) timeMs += now - lastTick; + else if (!video.seeking) { + timeMs = + activeClip.startMs + + (video.currentTime * 1000 - getClipSourceStartMs(activeClip)) / activeClip.speed; + } + timeMs = Math.min(duration(), timeMs); + lastTick = now; + sync(); + if (timeMs >= duration()) pause(); + else request = requestAnimationFrame(tick); + }; + return { + get isPlaying() { + return playing; + }, + play: async () => { + if (playing) return; + if (timeMs >= duration()) timeMs = 0; + playing = true; + onPlaying(true); + lastTick = performance.now(); + sync(true); + request = requestAnimationFrame(tick); + }, + pause, + seek: (seconds: number) => { + timeMs = Math.max(0, Math.min(duration(), seconds * 1000)); + lastTick = performance.now(); + sync(true); + }, + refresh: () => { + timeMs = Math.min(timeMs, duration()); + sync(true); + }, + dispose: pause, + }; +} diff --git a/src/components/video-editor/videoPlayback/index.ts b/src/components/video-editor/videoPlayback/index.ts index 06cf42edf..3913dc2a2 100644 --- a/src/components/video-editor/videoPlayback/index.ts +++ b/src/components/video-editor/videoPlayback/index.ts @@ -3,6 +3,5 @@ export * from "./focusUtils"; export * from "./layoutUtils"; export * from "./mathUtils"; export * from "./overlayUtils"; -export * from "./videoEventHandlers"; export * from "./zoomRegionUtils"; export * from "./zoomTransform"; diff --git a/src/components/video-editor/videoPlayback/sceneMotion.ts b/src/components/video-editor/videoPlayback/sceneMotion.ts index f749db304..7d4930aca 100644 --- a/src/components/video-editor/videoPlayback/sceneMotion.ts +++ b/src/components/video-editor/videoPlayback/sceneMotion.ts @@ -60,6 +60,7 @@ export function shouldComposePreviewFrame({ export function resolveSceneZoomTarget({ zoomRegions, timeMs, + cursorTimeMs = timeMs, connectZooms, zoomInDurationMs, zoomOutDurationMs, @@ -69,6 +70,7 @@ export function resolveSceneZoomTarget({ }: { zoomRegions: ZoomRegion[]; timeMs: number; + cursorTimeMs?: number; connectZooms?: boolean; zoomInDurationMs?: number; zoomOutDurationMs?: number; @@ -97,7 +99,7 @@ export function resolveSceneZoomTarget({ focus = computeCursorFollowFocus( cursorFollowCamera, cursorTelemetry, - timeMs, + cursorTimeMs, scale, strength, region.focus, diff --git a/src/components/video-editor/videoPlayback/videoEventHandlers.test.ts b/src/components/video-editor/videoPlayback/videoEventHandlers.test.ts deleted file mode 100644 index bf3fd5413..000000000 --- a/src/components/video-editor/videoPlayback/videoEventHandlers.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { createVideoEventHandlers } from "./videoEventHandlers"; - -type PresentedFrameCallback = (now: DOMHighResTimeStamp, metadata: { mediaTime?: number }) => void; - -type MockVideo = HTMLVideoElement & { - requestVideoFrameCallback?: (callback: PresentedFrameCallback) => number; - cancelVideoFrameCallback?: (handle: number) => void; -}; - -function createMutableRef(value: T) { - return { current: value }; -} - -function createMockVideo(overrides: Partial = {}): MockVideo { - const video = { - currentTime: 0.5, - duration: 10, - paused: false, - ended: false, - playbackRate: 1, - pause: vi.fn(), - } as unknown as MockVideo; - - return Object.assign(video, overrides); -} - -describe("createVideoEventHandlers", () => { - let requestAnimationFrameMock: ReturnType; - let cancelAnimationFrameMock: ReturnType; - - beforeEach(() => { - requestAnimationFrameMock = vi.fn(() => 11); - cancelAnimationFrameMock = vi.fn(); - vi.stubGlobal("requestAnimationFrame", requestAnimationFrameMock); - vi.stubGlobal("cancelAnimationFrame", cancelAnimationFrameMock); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it("prefers requestVideoFrameCallback mediaTime when available", () => { - let presentedFrameCallback: PresentedFrameCallback | null = null; - const video = createMockVideo({ - requestVideoFrameCallback: vi.fn((callback) => { - presentedFrameCallback = callback; - return 7; - }), - cancelVideoFrameCallback: vi.fn(), - }); - const onPlayStateChange = vi.fn(); - const onTimeUpdate = vi.fn(); - const currentTimeRef = createMutableRef(0); - const timeUpdateAnimationRef = createMutableRef(null); - - const handlers = createVideoEventHandlers({ - video, - isSeekingRef: createMutableRef(false), - isPlayingRef: createMutableRef(false), - allowPlaybackRef: createMutableRef(true), - currentTimeRef, - timeUpdateAnimationRef, - onPlayStateChange, - onTimeUpdate, - trimRegionsRef: createMutableRef([]), - speedRegionsRef: createMutableRef([]), - }); - - handlers.handlePlay(); - expect(onPlayStateChange).toHaveBeenCalledWith(true); - expect(video.requestVideoFrameCallback).toHaveBeenCalledTimes(1); - expect(requestAnimationFrameMock).not.toHaveBeenCalled(); - - presentedFrameCallback?.(0, { mediaTime: 1.25 }); - - expect(onTimeUpdate).toHaveBeenCalledWith(1.25); - expect(currentTimeRef.current).toBe(1250); - }); - - it("falls back to requestAnimationFrame when requestVideoFrameCallback is unavailable", () => { - let animationFrameCallback: FrameRequestCallback | null = null; - requestAnimationFrameMock.mockImplementation((callback: FrameRequestCallback) => { - animationFrameCallback = callback; - return 19; - }); - const video = createMockVideo({ currentTime: 0.75 }); - const onTimeUpdate = vi.fn(); - - const handlers = createVideoEventHandlers({ - video, - isSeekingRef: createMutableRef(false), - isPlayingRef: createMutableRef(false), - allowPlaybackRef: createMutableRef(true), - currentTimeRef: createMutableRef(0), - timeUpdateAnimationRef: createMutableRef(null), - onPlayStateChange: vi.fn(), - onTimeUpdate, - trimRegionsRef: createMutableRef([]), - speedRegionsRef: createMutableRef([]), - }); - - handlers.handlePlay(); - expect(requestAnimationFrameMock).toHaveBeenCalledTimes(1); - - video.paused = true; - animationFrameCallback?.(0); - - expect(onTimeUpdate).toHaveBeenCalledWith(0.75); - }); - - it("skips removed footage when playback reaches a cut region", () => { - let animationFrameCallback: FrameRequestCallback | null = null; - requestAnimationFrameMock.mockImplementation((callback: FrameRequestCallback) => { - animationFrameCallback = callback; - return 29; - }); - const video = createMockVideo({ currentTime: 1.25, duration: 10 }); - const onTimeUpdate = vi.fn(); - const handlers = createVideoEventHandlers({ - video, - isSeekingRef: createMutableRef(false), - isPlayingRef: createMutableRef(false), - allowPlaybackRef: createMutableRef(true), - currentTimeRef: createMutableRef(0), - timeUpdateAnimationRef: createMutableRef(null), - onPlayStateChange: vi.fn(), - onTimeUpdate, - trimRegionsRef: createMutableRef([{ id: "trim-1", startMs: 1000, endMs: 2000 }]), - speedRegionsRef: createMutableRef([]), - }); - - handlers.handlePlay(); - animationFrameCallback?.(0); - - expect(video.currentTime).toBe(2); - expect(video.pause).not.toHaveBeenCalled(); - expect(onTimeUpdate).toHaveBeenLastCalledWith(2); - }); - - it("cancels a pending requestVideoFrameCallback on pause and dispose", () => { - const cancelVideoFrameCallback = vi.fn(); - const video = createMockVideo({ - requestVideoFrameCallback: vi.fn(() => 23), - cancelVideoFrameCallback, - }); - const handlers = createVideoEventHandlers({ - video, - isSeekingRef: createMutableRef(false), - isPlayingRef: createMutableRef(false), - allowPlaybackRef: createMutableRef(true), - currentTimeRef: createMutableRef(0), - timeUpdateAnimationRef: createMutableRef(null), - onPlayStateChange: vi.fn(), - onTimeUpdate: vi.fn(), - trimRegionsRef: createMutableRef([]), - speedRegionsRef: createMutableRef([]), - }); - - handlers.handlePlay(); - handlers.handlePause(); - expect(cancelVideoFrameCallback).toHaveBeenCalledWith(23); - - cancelVideoFrameCallback.mockClear(); - handlers.handlePlay(); - handlers.dispose(); - expect(cancelVideoFrameCallback).toHaveBeenCalledWith(23); - }); - - it("skips removed footage after a paused seek", () => { - const video = createMockVideo({ - currentTime: 1.25, - paused: true, - }); - const onTimeUpdate = vi.fn(); - const shouldSnapPausedFrameRef = createMutableRef(false); - const handlers = createVideoEventHandlers({ - video, - isSeekingRef: createMutableRef(true), - shouldSnapPausedFrameRef, - isPlayingRef: createMutableRef(false), - allowPlaybackRef: createMutableRef(true), - currentTimeRef: createMutableRef(0), - timeUpdateAnimationRef: createMutableRef(null), - onPlayStateChange: vi.fn(), - onTimeUpdate, - trimRegionsRef: createMutableRef([{ id: "trim-1", startMs: 1000, endMs: 2000 }]), - speedRegionsRef: createMutableRef([]), - }); - - handlers.handleSeeking(); - handlers.handleSeeked(); - - expect(video.currentTime).toBe(2); - expect(onTimeUpdate).toHaveBeenLastCalledWith(2); - expect(shouldSnapPausedFrameRef.current).toBe(true); - }); -}); diff --git a/src/components/video-editor/videoPlayback/videoEventHandlers.ts b/src/components/video-editor/videoPlayback/videoEventHandlers.ts deleted file mode 100644 index 4b6be5d2c..000000000 --- a/src/components/video-editor/videoPlayback/videoEventHandlers.ts +++ /dev/null @@ -1,200 +0,0 @@ -import type React from "react"; -import { enablePitchPreservingPlayback } from "@/lib/mediaTiming"; -import type { SpeedRegion, TrimRegion } from "../types"; - -interface PresentedFrameMetadata { - mediaTime?: number; -} - -type PresentedFrameVideoElement = HTMLVideoElement & { - requestVideoFrameCallback?: ( - callback: (now: DOMHighResTimeStamp, metadata: PresentedFrameMetadata) => void, - ) => number; - cancelVideoFrameCallback?: (handle: number) => void; -}; - -interface VideoEventHandlersParams { - video: HTMLVideoElement; - isSeekingRef: React.MutableRefObject; - shouldSnapPausedFrameRef?: React.MutableRefObject; - isPlayingRef: React.MutableRefObject; - allowPlaybackRef: React.MutableRefObject; - currentTimeRef: React.MutableRefObject; - timeUpdateAnimationRef: React.MutableRefObject; - onPlayStateChange: (playing: boolean) => void; - onTimeUpdate: (time: number) => void; - trimRegionsRef: React.MutableRefObject; - speedRegionsRef: React.MutableRefObject; -} - -/** - * Bind media events to the preview's presented-frame clock while honoring trim - * and speed regions. - */ -export function createVideoEventHandlers(params: VideoEventHandlersParams) { - const { - video, - isSeekingRef, - shouldSnapPausedFrameRef, - isPlayingRef, - allowPlaybackRef, - currentTimeRef, - timeUpdateAnimationRef, - onPlayStateChange, - onTimeUpdate, - trimRegionsRef, - speedRegionsRef, - } = params; - const presentedFrameVideo = video as PresentedFrameVideoElement; - let videoFrameRequestId: number | null = null; - enablePitchPreservingPlayback(video); - - const emitTime = (timeValue: number) => { - currentTimeRef.current = timeValue * 1000; - onTimeUpdate(timeValue); - }; - - // Helper function to check if current time is within a trim region - const findActiveTrimRegion = (currentTimeMs: number): TrimRegion | null => { - const trimRegions = trimRegionsRef.current; - return ( - trimRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ) || null - ); - }; - - // Helper function to find the active speed region at the current time - const findActiveSpeedRegion = (currentTimeMs: number): SpeedRegion | null => { - return ( - speedRegionsRef.current.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ) || null - ); - }; - - const skipPastTrimRegion = (trimRegion: TrimRegion) => { - const skipToTime = trimRegion.endMs / 1000; - const clampedSkipToTime = Math.min(skipToTime, video.duration); - - video.currentTime = clampedSkipToTime; - emitTime(clampedSkipToTime); - - if (clampedSkipToTime >= video.duration) { - video.pause(); - } - }; - - const cancelScheduledUpdate = () => { - if (timeUpdateAnimationRef.current !== null) { - cancelAnimationFrame(timeUpdateAnimationRef.current); - timeUpdateAnimationRef.current = null; - } - - if ( - videoFrameRequestId !== null && - typeof presentedFrameVideo.cancelVideoFrameCallback === "function" - ) { - presentedFrameVideo.cancelVideoFrameCallback(videoFrameRequestId); - videoFrameRequestId = null; - } - }; - - const scheduleNextUpdate = () => { - if (video.paused || video.ended) { - return; - } - - // Align editor state with the frame Chromium actually presented instead of - // polling `currentTime` on a generic animation frame. - if (typeof presentedFrameVideo.requestVideoFrameCallback === "function") { - videoFrameRequestId = presentedFrameVideo.requestVideoFrameCallback( - (_now, metadata) => { - videoFrameRequestId = null; - updateTime(metadata); - }, - ); - return; - } - - timeUpdateAnimationRef.current = requestAnimationFrame(() => { - timeUpdateAnimationRef.current = null; - updateTime(); - }); - }; - - function getPresentedTime(metadata?: PresentedFrameMetadata): number { - const mediaTime = metadata?.mediaTime; - return Number.isFinite(mediaTime) ? (mediaTime ?? 0) : video.currentTime; - } - - function updateTime(metadata?: PresentedFrameMetadata) { - if (!video) return; - - const presentedTime = getPresentedTime(metadata); - const currentTimeMs = presentedTime * 1000; - const activeTrimRegion = findActiveTrimRegion(currentTimeMs); - - // If we're in a trim region during playback, skip to the end of it - if (activeTrimRegion && !video.paused && !video.ended) { - skipPastTrimRegion(activeTrimRegion); - } else { - // Apply playback speed from active speed region - const activeSpeedRegion = findActiveSpeedRegion(currentTimeMs); - enablePitchPreservingPlayback(video); - video.playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - emitTime(presentedTime); - } - - scheduleNextUpdate(); - } - - const handlePlay = () => { - if (!allowPlaybackRef.current) { - video.pause(); - return; - } - - isPlayingRef.current = true; - onPlayStateChange(true); - cancelScheduledUpdate(); - scheduleNextUpdate(); - }; - - const handlePause = () => { - isPlayingRef.current = false; - onPlayStateChange(false); - cancelScheduledUpdate(); - emitTime(video.currentTime); - }; - - const handleSeeked = () => { - isSeekingRef.current = false; - - const currentTimeMs = video.currentTime * 1000; - const activeTrimRegion = findActiveTrimRegion(currentTimeMs); - - // Never leave the preview parked on removed footage after a seek. - if (activeTrimRegion) { - skipPastTrimRegion(activeTrimRegion); - } else { - emitTime(video.currentTime); - } - }; - - const handleSeeking = () => { - isSeekingRef.current = true; - if (shouldSnapPausedFrameRef) { - shouldSnapPausedFrameRef.current = true; - } - emitTime(video.currentTime); - }; - - return { - dispose: cancelScheduledUpdate, - handlePlay, - handlePause, - handleSeeked, - handleSeeking, - }; -} diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 5cda0a885..26050c0a2 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,4 +1,5 @@ import type { WebDemuxer } from "web-demuxer"; +import { requiresClipTimelineRendering } from "./clipTimeline"; import type { AudioRegion, ClipRegion, @@ -149,6 +150,7 @@ export class AudioProcessor extends AudioTranscodeProcessor { // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( + requiresClipTimelineRendering(clipRegions) || sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0 || needsSourceAudioMixing || diff --git a/src/lib/exporter/audioProcessorShared.ts b/src/lib/exporter/audioProcessorShared.ts index ca6e35e95..a48062a78 100644 --- a/src/lib/exporter/audioProcessorShared.ts +++ b/src/lib/exporter/audioProcessorShared.ts @@ -109,12 +109,14 @@ export function hasNonDefaultSourceTrackSettings( } export interface TimelineSlice { + outputStartMs?: number; sourceStartMs: number; sourceEndMs: number; speed: number; } export interface PreparedOfflineRender { + usesClipTimeline?: boolean; mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; diff --git a/src/lib/exporter/audioTimelineProcessor.ts b/src/lib/exporter/audioTimelineProcessor.ts index 442537992..2f72c6e4c 100644 --- a/src/lib/exporter/audioTimelineProcessor.ts +++ b/src/lib/exporter/audioTimelineProcessor.ts @@ -87,6 +87,7 @@ export class AudioTimelineProcessor extends AudioProcessorBase { for (const slice of slices) { const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000; + if (slice.outputStartMs !== undefined) outputOffsetSec = slice.outputStartMs / 1000; const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed; // Where in the buffer does this slice read from? diff --git a/src/lib/exporter/clipAudioTimeline.test.ts b/src/lib/exporter/clipAudioTimeline.test.ts new file mode 100644 index 000000000..d6ef4bbcc --- /dev/null +++ b/src/lib/exporter/clipAudioTimeline.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi } from "vitest"; +import type { AudioRegion } from "@/components/video-editor/types"; +import { OfflineAudioProcessor } from "./offlineAudioProcessor"; + +const buffer = { duration: 12, numberOfChannels: 2 } as AudioBuffer; +class TestAudioProcessor extends OfflineAudioProcessor { + prepare = this.prepareOfflineRender.bind(this); + schedule = this.scheduleBufferThroughTimeline.bind(this); + scheduleOverlay = this.scheduleRegionForChunk.bind(this); + protected decodeAudioFromUrl = vi.fn(async () => buffer); + protected getMediaDurationSec = vi.fn(async () => 12); + protected stretchAudioBuffer = vi.fn(() => buffer); + get stretches() { + return this.stretchAudioBuffer.mock.calls; + } +} +const clips = [ + { id: "a", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 3 }, + { id: "b", startMs: 2000, endMs: 4000, sourceStartMs: 6000, speed: 3 }, +]; +function context() { + const starts: ReturnType[] = []; + return { + starts, + ctx: { + destination: {}, + createGain: () => ({ gain: { value: 1 }, connect: vi.fn() }), + createBufferSource: () => { + const start = vi.fn(); + starts.push(start); + return { playbackRate: { value: 1 }, connect: vi.fn(), start }; + }, + } as unknown as OfflineAudioContext, + }; +} + +describe("clip audio timeline", () => { + it("keeps the silent middle interval and schedules retained source at its timeline position", async () => { + const processor = new TestAudioProcessor(); + const prepared = await processor.prepare( + "file:///tmp/source.mp4", + [], + [], + [], + [], + undefined, + undefined, + clips, + ); + expect(prepared.outputDurationMs).toBe(4000); + expect(prepared.slices).toEqual([ + { sourceStartMs: 0, sourceEndMs: 3000, speed: 3, outputStartMs: 0 }, + { sourceStartMs: 6000, sourceEndMs: 12000, speed: 3, outputStartMs: 2000 }, + ]); + const { starts, ctx } = context(); + processor.schedule(ctx, buffer, prepared.slices, 0); + expect(starts.map((start) => start.mock.calls[0][0])).toEqual([0, 2]); + }); + it("keeps independently placed music at 1x across the video gap", () => { + const processor = new TestAudioProcessor(); + const { starts, ctx } = context(); + processor.scheduleOverlay( + ctx, + buffer, + { startMs: 500, endMs: 3500, volume: 1 } as AudioRegion, + [], + 0, + 4, + true, + ); + expect(starts[0]).toHaveBeenCalledWith(0.5, 0, 3); + }); + it("schedules a clip correctly when its audio straddles an offline chunk boundary", async () => { + const processor = new TestAudioProcessor(); + const prepared = await processor.prepare( + "file:///tmp/source.mp4", + [], + [], + [], + [], + undefined, + undefined, + clips, + ); + const { starts, ctx } = context(); + processor.schedule(ctx, buffer, prepared.slices, 0, 1, 2.5, 1); + expect(starts).toHaveLength(1); + expect(starts[0]).toHaveBeenCalledWith(0); + }); + it("an explicitly empty clip list schedules no source audio", async () => { + const processor = new TestAudioProcessor(); + const prepared = await processor.prepare( + "file:///tmp/source.mp4", + [], + [], + [], + [], + undefined, + undefined, + [], + ); + expect(prepared.slices).toEqual([]); + expect(prepared.outputDurationMs).toBe(12000); + }); +}); diff --git a/src/lib/exporter/clipTimeline.ts b/src/lib/exporter/clipTimeline.ts new file mode 100644 index 000000000..6bdb335aa --- /dev/null +++ b/src/lib/exporter/clipTimeline.ts @@ -0,0 +1,12 @@ +import { type ClipRegion, getClipSourceStartMs } from "@/components/video-editor/types"; + +/** Trim-only native paths concatenate source ranges; they cannot place clips. */ +export function requiresClipTimelineRendering(clips?: ClipRegion[]): boolean { + if (!clips) return false; + return ( + clips.length !== 1 || + clips[0].startMs !== 0 || + getClipSourceStartMs(clips[0]) !== 0 || + clips[0].speed !== 1 + ); +} diff --git a/src/lib/exporter/frameRenderer.test.ts b/src/lib/exporter/frameRenderer.test.ts index 56d5344e3..2ad9f704a 100644 --- a/src/lib/exporter/frameRenderer.test.ts +++ b/src/lib/exporter/frameRenderer.test.ts @@ -287,6 +287,20 @@ function createRenderer() { } describe("FrameRenderer webcam export path", () => { + it("draws the background without the screen or webcam during a timeline gap", async () => { + const renderer = createRenderer(); + const camera = { visible: true }; + const composite = vi.fn(); + Object.assign(renderer, { + app: { stage: {}, renderer: { render: vi.fn() } }, + cameraContainer: camera, + videoContainer: {}, + compositeWithShadows: composite, + }); + await renderer.renderFrame(null, 0, 0, 33333, 1500000); + expect(camera.visible).toBe(false); + expect(composite).toHaveBeenCalledWith(false); + }); const createdCanvases: ReturnType[] = []; beforeEach(() => { diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 473071d01..6d44f694f 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -80,6 +80,7 @@ import { buildTemporalSamplePlanUs, getTemporalMotionBlurConfig } from "./tempor const TEMPORAL_ZOOM_MOTION_BLUR_ENABLED = false; interface FrameRenderConfig { + timelineEffects?: boolean; width: number; height: number; preferredRenderBackend?: "webgl" | "webgpu"; @@ -1403,7 +1404,7 @@ export class FrameRenderer { } async renderFrame( - videoFrame: VideoFrame, + videoFrame: VideoFrame | null, timestamp: number, cursorTimestamp = timestamp, frameDurationUs?: number, @@ -1414,6 +1415,27 @@ export class FrameRenderer { } this.currentVideoTime = timestamp / 1000000; + this.cameraContainer.visible = videoFrame !== null; + if (!videoFrame) { + if (this.backgroundForwardFrameSource || this.backgroundVideoElement) { + await this.syncBackgroundFrame(backgroundTimelineTimestamp / 1_000_000); + } + this.app.renderer.render(this.app.stage); + this.compositeWithShadows(false); + if (this.compositeCtx && this.config.annotationRegions) { + await renderAnnotations( + this.compositeCtx, + this.config.annotationRegions, + this.config.width, + this.config.height, + backgroundTimelineTimestamp / 1000, + (this.config.width / BASE_PREVIEW_WIDTH + + this.config.height / BASE_PREVIEW_HEIGHT) / + 2, + ); + } + return; + } // Create or update video sprite from VideoFrame if (!this.videoSprite) { @@ -1487,7 +1509,7 @@ export class FrameRenderer { this.config.autoCaptionSettings, this.config.width, this.config.height, - temporalSnapshot.timeMs, + timestamp / 1000, ); } @@ -1504,7 +1526,9 @@ export class FrameRenderer { await this.syncBackgroundFrame(Math.max(0, backgroundTimelineTimestamp / 1_000_000)); } - const timeMs = this.currentVideoTime * 1000; + const timeMs = this.config.timelineEffects + ? backgroundTimelineTimestamp / 1000 + : timestamp / 1000; const cursorTimeMs = cursorTimestamp / 1000; if (this.cursorOverlay) { @@ -1520,7 +1544,7 @@ export class FrameRenderer { const TICKS_PER_FRAME = 1; for (let i = 0; i < TICKS_PER_FRAME; i++) { - this.updateAnimationState(timeMs); + this.updateAnimationState(timeMs, cursorTimeMs); } applyZoomTransform({ @@ -1593,7 +1617,7 @@ export class FrameRenderer { this.config.autoCaptionSettings, this.config.width, this.config.height, - timeMs, + timestamp / 1000, ); } } @@ -1662,12 +1686,13 @@ export class FrameRenderer { } /** Advance the export camera from the shared scene target at this media time. */ - private updateAnimationState(timeMs: number): number { + private updateAnimationState(timeMs: number, cursorTimeMs = timeMs): number { if (!this.cameraContainer || !this.layoutCache) return 0; const target = resolveSceneZoomTarget({ zoomRegions: this.config.zoomRegions, timeMs, + cursorTimeMs, connectZooms: this.config.connectZooms, zoomInDurationMs: this.config.zoomInDurationMs, zoomOutDurationMs: this.config.zoomOutDurationMs, @@ -1764,7 +1789,9 @@ export class FrameRenderer { await this.syncBackgroundFrame(Math.max(0, backgroundTimelineTimestamp / 1_000_000)); } - const timeMs = this.currentVideoTime * 1000; + const timeMs = this.config.timelineEffects + ? backgroundTimelineTimestamp / 1000 + : timestamp / 1000; const cursorTimeMs = cursorTimestamp / 1000; if (this.cursorOverlay) { @@ -1777,7 +1804,7 @@ export class FrameRenderer { ); } - this.updateAnimationState(timeMs); + this.updateAnimationState(timeMs, cursorTimeMs); applyZoomTransform({ cameraContainer: this.cameraContainer, @@ -1871,7 +1898,7 @@ export class FrameRenderer { return centerSnapshot ?? lastSnapshot; } - private compositeWithShadows(): void { + private compositeWithShadows(includeWebcam = true): void { if (!this.compositeCanvas || !this.compositeCtx || !this.app) return; const videoCanvas = this.app.canvas as HTMLCanvasElement; @@ -1927,7 +1954,7 @@ export class FrameRenderer { ctx.drawImage(videoCanvas, 0, 0, w, h); } - this.drawWebcamOverlay(ctx, w, h); + if (includeWebcam) this.drawWebcamOverlay(ctx, w, h); } private drawWebcamOverlay(ctx: CanvasRenderingContext2D, width: number, height: number): void { diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index d9e1b85e8..c7a0214df 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -30,6 +30,7 @@ const GIF_WORKER_URL = new URL("gif.js/dist/gif.worker.js", import.meta.url).toS const PROGRESS_SAMPLE_WINDOW_MS = 1_000; interface GifExporterConfig { + clipRegions?: import("@/components/video-editor/types").ClipRegion[]; videoUrl: string; width: number; height: number; @@ -139,6 +140,7 @@ export function buildGifFrameRendererConfig( ) { return { width: config.width, + timelineEffects: config.clipRegions !== undefined, height: config.height, wallpaper: config.wallpaper, zoomRegions: config.zoomRegions, @@ -254,6 +256,7 @@ export class GifExporter { const effectiveDuration = this.streamingDecoder.getEffectiveDuration( this.config.trimRegions, this.config.speedRegions, + this.config.clipRegions, ); const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); @@ -295,6 +298,7 @@ export class GifExporter { frameIndex++; this.reportProgress(frameIndex, totalFrames); }, + this.config.clipRegions, ); if (this.cancelled) { diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index dd820c81a..ce6aa9cc3 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -202,6 +202,29 @@ function createRenderer() { }); } +it("hides source layers in gaps while rendering timeline annotations at the output time", async () => { + const renderer = createRenderer(); + const camera = { visible: true }; + const webcam = { visible: true }; + const captions = { visible: true }; + const annotations = vi.fn(); + const renderOutput = vi.fn(async () => {}); + Object.assign(renderer, { + app: {}, + videoContainer: {}, + videoMaskGraphics: {}, + cameraContainer: camera, + webcamRootContainer: webcam, + captionContainer: captions, + updateAnnotationLayer: annotations, + renderOutput, + }); + await renderer.renderFrame(null, 0, 0, 33333, 1500000); + for (const layer of [camera, webcam, captions]) expect(layer.visible).toBe(false); + expect(annotations).toHaveBeenCalledWith(1500); + expect(renderOutput).toHaveBeenCalledWith(1500); +}); + describe("ModernFrameRenderer Pixi lifecycle", () => { it("continues to the next backend when failed-init cleanup would throw", async () => { pixiApplicationInstancesMock.length = 0; diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 25dc33c39..9dee18985 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -101,6 +101,7 @@ const TEMPORAL_ZOOM_MOTION_BLUR_ENABLED = false; import type { ExportRenderBackend } from "./types"; interface FrameRenderConfig { + timelineEffects?: boolean; width: number; height: number; preferredRenderBackend?: ExportRenderBackend; @@ -2930,7 +2931,9 @@ export class FrameRenderer { await this.syncBackgroundFrame(Math.max(0, backgroundTimelineTimestamp / 1_000_000)); } - const timeMs = this.currentVideoTime * 1000; + const timeMs = this.config.timelineEffects + ? backgroundTimelineTimestamp / 1000 + : timestamp / 1000; const cursorTimeMs = cursorTimestamp / 1000; if (this.cursorOverlay) { @@ -2943,7 +2946,7 @@ export class FrameRenderer { ); } - this.updateAnimationState(timeMs); + this.updateAnimationState(timeMs, cursorTimeMs); applyZoomTransform({ cameraContainer: this.cameraContainer, @@ -2969,7 +2972,7 @@ export class FrameRenderer { if (includeOverlayLayers) { this.updateAnnotationLayer(timeMs); - this.updateCaptionLayer(timeMs); + this.updateCaptionLayer(timestamp / 1000); } this.updateWebcamOverlay(webcamRenderTimeSeconds); @@ -3083,7 +3086,7 @@ export class FrameRenderer { return null; } - this.updateCaptionLayer(resolvedSnapshot.timeMs); + this.updateCaptionLayer(timestamp / 1000); const hasOverlayCanvasWork = (this.config.annotationRegions?.length ?? 0) > 0 || @@ -3102,7 +3105,7 @@ export class FrameRenderer { } async renderFrame( - videoFrame: VideoFrame, + videoFrame: VideoFrame | null, timestamp: number, cursorTimestamp = timestamp, frameDurationUs?: number, @@ -3113,6 +3116,18 @@ export class FrameRenderer { } this.currentVideoTime = timestamp / 1_000_000; + this.cameraContainer.visible = videoFrame !== null; + if (!videoFrame) { + if (this.backgroundForwardFrameSource || this.backgroundVideoElement) { + await this.syncBackgroundFrame(backgroundTimelineTimestamp / 1_000_000); + } + if (this.webcamRootContainer) this.webcamRootContainer.visible = false; + if (this.captionContainer) this.captionContainer.visible = false; + this.updateAnnotationLayer(backgroundTimelineTimestamp / 1000); + await this.renderOutput(backgroundTimelineTimestamp / 1000); + return; + } + if (this.captionContainer) this.captionContainer.visible = true; const resolvedVideoSource = await this.resolveDetachedVideoFrameSource( videoFrame, @@ -3173,7 +3188,9 @@ export class FrameRenderer { await this.syncBackgroundFrame(Math.max(0, backgroundTimelineTimestamp / 1_000_000)); } - const timeMs = this.currentVideoTime * 1000; + const timeMs = this.config.timelineEffects + ? backgroundTimelineTimestamp / 1000 + : timestamp / 1000; const cursorTimeMs = cursorTimestamp / 1000; if (this.cursorOverlay) { @@ -3186,7 +3203,7 @@ export class FrameRenderer { ); } - this.updateAnimationState(timeMs); + this.updateAnimationState(timeMs, cursorTimeMs); applyZoomTransform({ cameraContainer: this.cameraContainer, @@ -3211,9 +3228,12 @@ export class FrameRenderer { }); this.updateAnnotationLayer(timeMs); - this.updateCaptionLayer(timeMs); + this.updateCaptionLayer(timestamp / 1000); this.updateWebcamOverlay(); + await this.renderOutput(timeMs); + } + private async renderOutput(timeMs: number): Promise { if (this.hasActiveBlurAnnotations(timeMs)) { const annotationContainerVisible = this.annotationContainer?.visible ?? true; const captionContainerVisible = this.captionContainer?.visible ?? true; @@ -3225,7 +3245,7 @@ export class FrameRenderer { this.captionContainer.visible = false; } - this.app.render(); + this.app!.render(); if (this.annotationContainer) { this.annotationContainer.visible = annotationContainerVisible; @@ -3239,7 +3259,7 @@ export class FrameRenderer { } this.outputCanvasOverride = null; - this.app.render(); + this.app!.render(); } private updateLayout(): void { @@ -3344,7 +3364,7 @@ export class FrameRenderer { } /** Advance the export camera from the shared scene target at this media time. */ - private updateAnimationState(timeMs: number): number { + private updateAnimationState(timeMs: number, cursorTimeMs = timeMs): number { if (!this.cameraContainer || !this.layoutCache) { return 0; } @@ -3352,6 +3372,7 @@ export class FrameRenderer { const target = resolveSceneZoomTarget({ zoomRegions: this.config.zoomRegions, timeMs, + cursorTimeMs, connectZooms: this.config.connectZooms, zoomInDurationMs: this.config.zoomInDurationMs, zoomOutDurationMs: this.config.zoomOutDurationMs, diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 5fecd41a5..1b9326e2f 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -1,3 +1,4 @@ +import { requiresClipTimelineRendering } from "./clipTimeline"; import type { AnnotationRegion, AudioRegion, @@ -545,10 +546,14 @@ export class ModernVideoExporter { const shouldUseFfmpegAudioFallback = !useNativeEncoder && nativeAudioPlan.audioMode !== "none" && - (shouldUsePitchPreservingFfmpegAudio || !(await isAacAudioEncodingSupported())); + // The PCM/FFmpeg path preserves AAC priming; WebCodecs AAC can shift clip cuts. + (requiresClipTimelineRendering(this.config.clipRegions) || + shouldUsePitchPreservingFfmpegAudio || + !(await isAacAudioEncodingSupported())); const effectiveDuration = this.streamingDecoder.getEffectiveDuration( this.config.trimRegions, this.config.speedRegions, + this.config.clipRegions, ); this.effectiveDurationSec = effectiveDuration; const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); @@ -597,6 +602,7 @@ export class ModernVideoExporter { stageStartedAt = this.getNowMs(); this.renderer = new ModernFrameRenderer({ + timelineEffects: this.config.clipRegions !== undefined, width: this.config.width, height: this.config.height, preferredRenderBackend: undefined, @@ -731,6 +737,7 @@ export class ModernVideoExporter { this.processedFrameCount = frameIndex; this.reportProgress(frameIndex, totalFrames, "extracting"); }, + this.config.clipRegions, ); this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt; @@ -1464,6 +1471,7 @@ export class ModernVideoExporter { } if ( + requiresClipTimelineRendering(this.config.clipRegions) || speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || @@ -1490,6 +1498,7 @@ export class ModernVideoExporter { Number.isFinite(primaryAudioSourceSampleRate) && primaryAudioSourceSampleRate > 0; const requiresRenderedEditedTrack = + requiresClipTimelineRendering(this.config.clipRegions) || hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)); const strategy = @@ -1670,6 +1679,8 @@ export class ModernVideoExporter { effectiveDurationSec: number, ): string[] { const reasons: string[] = []; + if (requiresClipTimelineRendering(this.config.clipRegions)) + reasons.push("explicit-clip-timeline"); if ( typeof window === "undefined" || !window.electronAPI?.nativeStaticLayoutExport || diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts index 7526db08b..4e60597d3 100644 --- a/src/lib/exporter/offlineAudioProcessor.ts +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -1,4 +1,9 @@ import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; +import { + getClipSourceStartMs, + getClipSourceEndMs, + getTimelineDurationMs, +} from "@/components/video-editor/types"; import type { AudioRegion, ClipRegion, @@ -149,16 +154,26 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { const sourceDurationMs = sourceDurationSec * 1000; // Build timeline slices (non-trimmed segments with speed info) - const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions); + const slices = clipRegions + ? clipRegions.map((clip) => ({ + sourceStartMs: getClipSourceStartMs(clip), + sourceEndMs: getClipSourceEndMs(clip), + speed: clip.speed, + outputStartMs: clip.startMs, + })) + : this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions); let outputDurationMs = 0; for (const slice of slices) { outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed; } + if (clipRegions) outputDurationMs = getTimelineDurationMs(clipRegions, sourceDurationMs); // Extend for audio regions that might exceed the video timeline for (const { region } of regionEntries) { - const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices); + const regionEndOutput = clipRegions + ? region.endMs + : this.sourceTimeToOutputTime(region.endMs, slices); outputDurationMs = Math.max(outputDurationMs, regionEndOutput); } @@ -177,6 +192,7 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { })); return { + usesClipTimeline: clipRegions !== undefined, mainBufferEntry, companionEntries, regionEntries, @@ -341,6 +357,7 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { slices, outputOffsetSec, chunkSec, + prepared.usesClipTimeline, ); } @@ -363,9 +380,14 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { slices: TimelineSlice[], chunkOutputStartSec: number, chunkDurationSec: number, + usesClipTimeline = false, ): void { - const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices); - const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices); + const outputStartMs = usesClipTimeline + ? region.startMs + : this.sourceTimeToOutputTime(region.startMs, slices); + const outputEndMs = usesClipTimeline + ? region.endMs + : this.sourceTimeToOutputTime(region.endMs, slices); let localStartSec = outputStartMs / 1000 - chunkOutputStartSec; let localEndSec = outputEndMs / 1000 - chunkOutputStartSec; diff --git a/src/lib/exporter/streamingDecodePipeline.ts b/src/lib/exporter/streamingDecodePipeline.ts index 289bcf380..6ed2298dc 100644 --- a/src/lib/exporter/streamingDecodePipeline.ts +++ b/src/lib/exporter/streamingDecodePipeline.ts @@ -8,7 +8,13 @@ import { type DecodedVideoInfo, type VideoDecodeFailureContext, } from "./streamingDecoderSupport"; -import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments"; +import { + computeVideoSegments, + splitVideoSegmentsBySpeed, + segmentFrameCount, + segmentSourceTime, + type VideoSegment, +} from "./videoTimelineSegments"; type OnFrameCallback = ( frame: VideoFrame, @@ -38,6 +44,7 @@ export async function decodeVideoStream( trimRegions: TrimRegion[] | undefined, speedRegions: SpeedRegion[] | undefined, onFrame: OnFrameCallback, + segmentsOverride?: VideoSegment[], ): Promise { if (!context.demuxer || !context.metadata) { throw new Error("Must call loadMetadata() before decodeAll()"); @@ -50,12 +57,14 @@ export async function decodeVideoStream( duration: context.metadata.duration, streamDuration: context.metadata.streamDuration, }); - const segments = splitVideoSegmentsBySpeed( - computeVideoSegments(effectiveVideoDuration, trimRegions), - speedRegions, - ); + const segments = + segmentsOverride ?? + splitVideoSegmentsBySpeed( + computeVideoSegments(effectiveVideoDuration, trimRegions), + speedRegions, + ); const segmentOutputFrameCounts = segments.map((segment) => - Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate), + segmentFrameCount(segment, targetFrameRate), ); const expectedOutputFrames = segmentOutputFrameCounts.reduce((sum, count) => sum + count, 0); const frameDurationUs = 1_000_000 / targetFrameRate; @@ -256,24 +265,20 @@ export async function decodeVideoStream( let heldFrame: VideoFrame | null = null; let heldFrameSec = 0; - const emitHeldFrameForTarget = async (segment: { - startSec: number; - endSec: number; - speed: number; - }) => { + const emitHeldFrameForTarget = async (segment: VideoSegment) => { if (!heldFrame) return false; const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; if (segmentFrameIndex >= segmentFrameCount) return false; - const segmentDurationSec = segment.endSec - segment.startSec; - const sourceTimeSec = - segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; - if (sourceTimeSec >= segment.endSec - epsilonSec) return false; + const sourceTimeSec = segmentSourceTime(segment, segmentFrameIndex, targetFrameRate); const sourceTimestampMs = sourceTimeSec * 1000; await onFrame( heldFrame, - exportFrameIndex * frameDurationUs, + (segment.outputStartSec === undefined + ? exportFrameIndex + : Math.ceil(segment.outputStartSec * targetFrameRate) + segmentFrameIndex) * + frameDurationUs, sourceTimestampMs, sourceTimestampMs, ); @@ -354,26 +359,16 @@ export async function decodeVideoStream( break; } - const segmentDurationSec = currentSegment.endSec - currentSegment.startSec; - const sourceTimeSec = - currentSegment.startSec + - (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; - if (sourceTimeSec >= currentSegment.endSec - epsilonSec) { - break; - } + const sourceTimeSec = segmentSourceTime( + currentSegment, + segmentFrameIndex, + targetFrameRate, + ); if (sourceTimeSec > handoffBoundarySec) { break; } - const sourceTimestampMs = sourceTimeSec * 1000; - await onFrame( - heldFrame, - exportFrameIndex * frameDurationUs, - sourceTimestampMs, - sourceTimestampMs, - ); - segmentFrameIndex++; - exportFrameIndex++; + await emitHeldFrameForTarget(currentSegment); } heldFrame.close(); diff --git a/src/lib/exporter/streamingDecoder.test.ts b/src/lib/exporter/streamingDecoder.test.ts index 11805fd02..542653a2f 100644 --- a/src/lib/exporter/streamingDecoder.test.ts +++ b/src/lib/exporter/streamingDecoder.test.ts @@ -202,6 +202,65 @@ describe("StreamingVideoDecoder decode failures", () => { expect(onFrame).not.toHaveBeenCalled(); expect(frame.close).toHaveBeenCalledTimes(1); }); + + it.each([ + false, + true, + ])("emits real gap frames on the output clock (reordered: %s)", async (reordered) => { + mockDemuxerRead.mockImplementation( + () => + new ReadableStream({ + start(controller) { + for (let i = 0; i < 120; i++) + controller.enqueue({ timestamp: (i * 1_000_000) / 30 }); + controller.close(); + }, + }), + ); + class TestDecoder { + state = "unconfigured"; + decodeQueueSize = 0; + constructor(private callbacks: { output: (frame: VideoFrame) => void }) {} + configure() { + this.state = "configured"; + } + decode(chunk: EncodedVideoChunk) { + this.callbacks.output({ + timestamp: chunk.timestamp, + close: vi.fn(), + } as unknown as VideoFrame); + } + async flush() {} + close() { + this.state = "closed"; + } + } + vi.stubGlobal("VideoDecoder", TestDecoder); + const decoder = new StreamingVideoDecoder(); + await decoder.loadMetadata("/tmp/clip-timeline.mp4"); + const clips = [ + { id: "a", startMs: 0, endMs: 400, sourceStartMs: reordered ? 2400 : 0, speed: 3 }, + { id: "b", startMs: 800, endMs: 1200, sourceStartMs: reordered ? 0 : 2400, speed: 3 }, + ]; + const frames: Array<{ gap: boolean; timestamp: number; source: number }> = []; + await decoder.decodeAll( + 30, + undefined, + undefined, + async (frame, timestamp, source) => { + frames.push({ gap: frame === null, timestamp, source }); + }, + clips, + ); + expect(frames).toHaveLength(36); + expect(frames.filter((f) => f.gap)).toHaveLength(12); + for (let i = 0; i < frames.length; i++) { + expect(frames[i].timestamp).toBeCloseTo((i * 1_000_000) / 30, 5); + expect(frames[i].gap).toBe(i >= 12 && i < 24); + } + expect(frames[24].source).toBeCloseTo(reordered ? 0 : 2400); + expect(decoder.getEffectiveDuration(undefined, undefined, clips)).toBe(1.2); + }); }); describe("StreamingVideoDecoder local media loading", () => { diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index 152fdb19b..8ba35ea40 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -1,9 +1,19 @@ import { WebDemuxer } from "web-demuxer"; -import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; +import { + type ClipRegion, + type SpeedRegion, + type TrimRegion, + getTimelineDurationMs, + findClipAtTimelineTime, +} from "@/components/video-editor/types"; import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; import { createFallbackDemuxerSource, resolveMediaResourceUrl } from "./localMediaSource"; import { decodeVideoStream } from "./streamingDecodePipeline"; -import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments"; +import { + buildClipDecodeRuns, + computeVideoSegments, + splitVideoSegmentsBySpeed, +} from "./videoTimelineSegments"; const DEFAULT_MAX_DECODE_QUEUE = 12; const DEFAULT_MAX_PENDING_FRAMES = 32; @@ -25,7 +35,7 @@ interface StreamingVideoDecoderLoadOptions { /** Decoder retains ownership of the VideoFrame and closes it after use. */ type OnFrameCallback = ( - frame: VideoFrame, + frame: VideoFrame | null, exportTimestampUs: number, sourceTimestampMs: number, cursorTimestampMs: number, @@ -167,38 +177,74 @@ export class StreamingVideoDecoder { trimRegions: TrimRegion[] | undefined, speedRegions: SpeedRegion[] | undefined, onFrame: OnFrameCallback, + clipRegions?: ClipRegion[], ): Promise { if (!this.demuxer || !this.metadata) { throw new Error("Must call loadMetadata() before decodeAll()"); } const owner = this; - await decodeVideoStream( - { - demuxer: this.demuxer, - metadata: this.metadata, - pendingFrames: this.pendingFrames, - maxDecodeQueue: this.maxDecodeQueue, - maxPendingFrames: this.maxPendingFrames, - get cancelled() { - return owner.cancelled; - }, - get decoder() { - return owner.decoder; - }, - set decoder(value) { - owner.decoder = value; - }, + const context = { + demuxer: this.demuxer, + metadata: this.metadata, + pendingFrames: this.pendingFrames, + maxDecodeQueue: this.maxDecodeQueue, + maxPendingFrames: this.maxPendingFrames, + get cancelled() { + return owner.cancelled; }, - targetFrameRate, - trimRegions, - speedRegions, - onFrame, + get decoder() { + return owner.decoder; + }, + set decoder(value) { + owner.decoder = value; + }, + }; + if (!clipRegions) { + await decodeVideoStream(context, targetFrameRate, trimRegions, speedRegions, onFrame); + return; + } + let nextFrame = 0; + const emitGapsUntil = async (endFrame: number) => { + while (!this.cancelled && nextFrame < endFrame) { + if (findClipAtTimelineTime((nextFrame * 1000) / targetFrameRate, clipRegions)) { + throw new Error(`Missing decoded clip frame at output frame ${nextFrame}`); + } + await onFrame(null, (nextFrame * 1_000_000) / targetFrameRate, 0, 0); + nextFrame++; + } + }; + for (const run of buildClipDecodeRuns(clipRegions)) { + if (this.cancelled) break; + await decodeVideoStream( + context, + targetFrameRate, + undefined, + undefined, + async (frame, timestamp, source, cursor) => { + await emitGapsUntil(Math.round((timestamp * targetFrameRate) / 1_000_000)); + if (this.cancelled) return; + await onFrame(frame, timestamp, source, cursor); + nextFrame++; + }, + run, + ); + } + await emitGapsUntil( + Math.ceil( + this.getEffectiveDuration(undefined, undefined, clipRegions) * targetFrameRate, + ), ); } - getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number { + getEffectiveDuration( + trimRegions?: TrimRegion[], + speedRegions?: SpeedRegion[], + clipRegions?: ClipRegion[], + ): number { if (!this.metadata) throw new Error("Must call loadMetadata() first"); + if (clipRegions) + return getTimelineDurationMs(clipRegions, this.metadata.duration * 1000) / 1000; const trimSegments = computeVideoSegments( getEffectiveVideoStreamDurationSeconds({ duration: this.metadata.duration, diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 9349627c5..d8f1c6c0f 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -1,3 +1,4 @@ +import { requiresClipTimelineRendering } from "./clipTimeline"; import type { AnnotationRegion, AudioRegion, @@ -206,7 +207,9 @@ export class VideoExporter { const shouldUseFfmpegAudioFallback = !useNativeEncoder && audioPlan.audioMode !== "none" && - (shouldUsePitchPreservingFfmpegAudio || !(await isAacAudioEncodingSupported())); + (requiresClipTimelineRendering(this.config.clipRegions) || + shouldUsePitchPreservingFfmpegAudio || + !(await isAacAudioEncodingSupported())); if (!useNativeEncoder) { await this.initializeEncoder(); @@ -214,6 +217,7 @@ export class VideoExporter { // Initialize frame renderer this.renderer = new FrameRenderer({ + timelineEffects: this.config.clipRegions !== undefined, width: this.config.width, height: this.config.height, preferredRenderBackend: undefined, @@ -281,6 +285,7 @@ export class VideoExporter { const effectiveDuration = this.streamingDecoder.getEffectiveDuration( this.config.trimRegions, this.config.speedRegions, + this.config.clipRegions, ); this.effectiveDurationSec = effectiveDuration; const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); @@ -326,6 +331,7 @@ export class VideoExporter { this.processedFrameCount = frameIndex; this.reportProgress(frameIndex, totalFrames); }, + this.config.clipRegions, ); if (this.cancelled) { @@ -571,6 +577,7 @@ export class VideoExporter { } if ( + requiresClipTimelineRendering(this.config.clipRegions) || speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || @@ -589,6 +596,7 @@ export class VideoExporter { ); const trimRegions = this.config.trimRegions ?? []; const canUsePrimaryAudioFiltergraph = + !requiresClipTimelineRendering(this.config.clipRegions) && Boolean(primaryAudioSourcePath) && !hasTimedSourceAudioFallback && (usesEmbeddedPrimaryAudio || diff --git a/src/lib/exporter/videoTimelineSegments.test.ts b/src/lib/exporter/videoTimelineSegments.test.ts new file mode 100644 index 000000000..090270077 --- /dev/null +++ b/src/lib/exporter/videoTimelineSegments.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { buildClipDecodeRuns, segmentFrameCount, segmentSourceTime } from "./videoTimelineSegments"; +import { requiresClipTimelineRendering } from "./clipTimeline"; + +describe("explicit clip export timeline", () => { + it("preserves gaps and source in-points without starting another decode pass", () => { + const runs = buildClipDecodeRuns([ + { id: "a", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 3 }, + { id: "b", startMs: 2000, endMs: 4000, sourceStartMs: 6000, speed: 3 }, + ]); + expect(runs).toEqual([ + [ + { startSec: 0, endSec: 3, speed: 3, outputStartSec: 0, outputEndSec: 1 }, + { startSec: 6, endSec: 12, speed: 3, outputStartSec: 2, outputEndSec: 4 }, + ], + ]); + }); + it("starts a new source pass for reordered footage", () => { + const runs = buildClipDecodeRuns([ + { id: "b", startMs: 2000, endMs: 3000, sourceStartMs: 0, speed: 1 }, + { id: "a", startMs: 0, endMs: 1000, sourceStartMs: 5000, speed: 1 }, + ]); + expect(runs.map((run) => run[0].startSec)).toEqual([5, 0]); + }); + it.each([ + 24, 30, 60, 59.94, + ])("uses a single %s fps grid across fractional-frame cuts", (fps) => { + const clips = Array.from({ length: 100 }, (_, i) => ({ + id: `${i}`, + startMs: i * 137, + endMs: (i + 1) * 137, + sourceStartMs: i * 411, + speed: 3, + })); + const [segments] = buildClipDecodeRuns(clips); + expect(segments.reduce((n, s) => n + segmentFrameCount(s, fps), 0)).toBe( + Math.ceil(13.7 * fps), + ); + for (const segment of segments) { + for (let i = 0; i < segmentFrameCount(segment, fps); i++) { + const source = segmentSourceTime(segment, i, fps); + expect(source).toBeGreaterThanOrEqual(segment.startSec); + expect(source).toBeLessThan(segment.endSec); + } + } + }); + it("does not route positioned clips through a concatenating native path", () => { + expect(requiresClipTimelineRendering(undefined)).toBe(false); + expect( + requiresClipTimelineRendering([{ id: "full", startMs: 0, endMs: 1000, speed: 1 }]), + ).toBe(false); + expect(requiresClipTimelineRendering([])).toBe(true); + expect( + requiresClipTimelineRendering([{ id: "gap", startMs: 1000, endMs: 2000, speed: 1 }]), + ).toBe(true); + }); +}); diff --git a/src/lib/exporter/videoTimelineSegments.ts b/src/lib/exporter/videoTimelineSegments.ts index 9cb198223..1f4bc411a 100644 --- a/src/lib/exporter/videoTimelineSegments.ts +++ b/src/lib/exporter/videoTimelineSegments.ts @@ -1,4 +1,51 @@ -import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; +import { + type ClipRegion, + type SpeedRegion, + type TrimRegion, + getClipSourceStartMs, + getClipSourceEndMs, + sortClipRegions, +} from "@/components/video-editor/types"; + +export interface VideoSegment { + startSec: number; + endSec: number; + speed: number; + outputStartSec?: number; + outputEndSec?: number; +} + +/** One forward decode pass per source-order run; timeline gaps need no decoding. */ +export function buildClipDecodeRuns(clips: ClipRegion[]): VideoSegment[][] { + const runs: VideoSegment[][] = []; + for (const clip of sortClipRegions(clips)) { + const segment: VideoSegment = { + startSec: getClipSourceStartMs(clip) / 1000, + endSec: getClipSourceEndMs(clip) / 1000, + speed: clip.speed, + outputStartSec: clip.startMs / 1000, + outputEndSec: clip.endMs / 1000, + }; + const run = runs[runs.length - 1]; + if (run && segment.startSec >= run[run.length - 1].endSec) run.push(segment); + else runs.push([segment]); + } + return runs; +} + +export function segmentFrameCount(segment: VideoSegment, fps: number): number { + return segment.outputStartSec !== undefined && segment.outputEndSec !== undefined + ? Math.ceil(segment.outputEndSec * fps) - Math.ceil(segment.outputStartSec * fps) + : Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * fps); +} + +export function segmentSourceTime(segment: VideoSegment, index: number, fps: number): number { + const outputOffset = + segment.outputStartSec === undefined + ? index / fps + : (Math.ceil(segment.outputStartSec * fps) + index) / fps - segment.outputStartSec; + return segment.startSec + outputOffset * segment.speed; +} export function computeVideoSegments( totalDuration: number, From cd2405f2b2a8438072d474197e1acdd80879ceb4 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:30:28 +1000 Subject: [PATCH 04/10] Render clip gaps as solid black --- src/components/video-editor/VideoPlayback.tsx | 17 +++++++++++---- src/lib/exporter/frameRenderer.test.ts | 15 ++++++++----- src/lib/exporter/frameRenderer.ts | 21 +++---------------- src/lib/exporter/modernFrameRenderer.test.ts | 19 ++++++++++------- src/lib/exporter/modernFrameRenderer.ts | 12 +++++------ 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 3c722e06f..33b1c6793 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -46,6 +46,7 @@ import { type AnnotationRegion, type AutoCaptionSettings, type CaptionCue, + type ClipRegion, type CursorClickEffectStyle, type CursorStyle, DEFAULT_CONNECTED_ZOOM_DURATION_MS, @@ -75,7 +76,9 @@ import { DEFAULT_ZOOM_MOTION_BLUR_TUNING, DEFAULT_ZOOM_OUT_DURATION_MS, DEFAULT_ZOOM_OUT_EASING, + findClipAtTimelineTime, getDefaultCaptionFontFamily, + mapTimelineTimeToSourceTime, type Padding, type WebcamOverlaySettings, type ZoomDepth, @@ -88,6 +91,7 @@ import { isAnnotationActiveAtTime, shouldClearSelectedAnnotation, } from "./videoPlayback/annotationVisibility"; +import { createClipPlayback } from "./videoPlayback/clipPlayback"; import { DEFAULT_FOCUS } from "./videoPlayback/constants"; import { type CursorFollowCameraState, @@ -116,8 +120,6 @@ import { resolveSceneZoomTarget, shouldComposePreviewFrame, } from "./videoPlayback/sceneMotion"; -import { createClipPlayback } from "./videoPlayback/clipPlayback"; -import { type ClipRegion, findClipAtTimelineTime, mapTimelineTimeToSourceTime } from "./types"; import { getWebcamMediaTargetTimeSeconds, isWebcamMediaSynchronized, @@ -2474,6 +2476,7 @@ const VideoPlayback = forwardRef( style={{ width: "100%", aspectRatio: formatAspectRatioForCSS(aspectRatio, nativeAspectRatio), + backgroundColor: "#000000", borderRadius: 0, clipPath: "none", }} @@ -2489,6 +2492,7 @@ const VideoPlayback = forwardRef( loop playsInline style={{ + visibility: isGap ? "hidden" : "visible", filter: sceneEffects.backgroundBlurPx > 0 ? `blur(${sceneEffects.backgroundBlurPx}px)` @@ -2503,6 +2507,7 @@ const VideoPlayback = forwardRef( className="absolute inset-0 bg-cover bg-center" style={{ ...backgroundStyle, + visibility: isGap ? "hidden" : "visible", filter: sceneEffects.backgroundBlurPx > 0 ? `blur(${sceneEffects.backgroundBlurPx}px)` @@ -2519,7 +2524,7 @@ const VideoPlayback = forwardRef( visibility: isGap ? "hidden" : "visible", }} /> - {hasRendererFallback && ( + {hasRendererFallback && !isGap && (
{`Pixi renderer unavailable on this environment (${pixiRendererBackend ?? "unknown"}).`} @@ -2534,7 +2539,10 @@ const VideoPlayback = forwardRef(
( ref={attachVideo} src={videoPath} className={fallbackVideoClassName} + style={{ visibility: isGap ? "hidden" : "visible" }} preload="metadata" playsInline aria-hidden="true" diff --git a/src/lib/exporter/frameRenderer.test.ts b/src/lib/exporter/frameRenderer.test.ts index 2ad9f704a..ba1701e53 100644 --- a/src/lib/exporter/frameRenderer.test.ts +++ b/src/lib/exporter/frameRenderer.test.ts @@ -112,6 +112,7 @@ type MockContext = { scale: MockFunction; clearRect: MockFunction; filter: string; + fillStyle: string; }; type MockCanvas = ReturnType; type FrameRendererTestAccess = { @@ -251,6 +252,7 @@ function createMockContext() { scale: vi.fn(), clearRect: vi.fn(), filter: "", + fillStyle: "", }; } @@ -287,19 +289,22 @@ function createRenderer() { } describe("FrameRenderer webcam export path", () => { - it("draws the background without the screen or webcam during a timeline gap", async () => { + it("renders a timeline gap as solid black", async () => { const renderer = createRenderer(); const camera = { visible: true }; - const composite = vi.fn(); + const context = createMockContext(); + const app = { stage: {}, renderer: { render: vi.fn() } }; Object.assign(renderer, { - app: { stage: {}, renderer: { render: vi.fn() } }, + app, cameraContainer: camera, videoContainer: {}, - compositeWithShadows: composite, + compositeCtx: context, }); await renderer.renderFrame(null, 0, 0, 33333, 1500000); expect(camera.visible).toBe(false); - expect(composite).toHaveBeenCalledWith(false); + expect(context.fillStyle).toBe("#000000"); + expect(context.fillRect).toHaveBeenCalledWith(0, 0, 1920, 1080); + expect(app.renderer.render).not.toHaveBeenCalled(); }); const createdCanvases: ReturnType[] = []; diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 6d44f694f..41cc9d627 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -1410,30 +1410,15 @@ export class FrameRenderer { frameDurationUs?: number, backgroundTimelineTimestamp = timestamp, ): Promise { - if (!this.app || !this.videoContainer || !this.cameraContainer) { + if (!this.app || !this.videoContainer || !this.cameraContainer || !this.compositeCtx) { throw new Error("Renderer not initialized"); } this.currentVideoTime = timestamp / 1000000; this.cameraContainer.visible = videoFrame !== null; if (!videoFrame) { - if (this.backgroundForwardFrameSource || this.backgroundVideoElement) { - await this.syncBackgroundFrame(backgroundTimelineTimestamp / 1_000_000); - } - this.app.renderer.render(this.app.stage); - this.compositeWithShadows(false); - if (this.compositeCtx && this.config.annotationRegions) { - await renderAnnotations( - this.compositeCtx, - this.config.annotationRegions, - this.config.width, - this.config.height, - backgroundTimelineTimestamp / 1000, - (this.config.width / BASE_PREVIEW_WIDTH + - this.config.height / BASE_PREVIEW_HEIGHT) / - 2, - ); - } + this.compositeCtx.fillStyle = "#000000"; + this.compositeCtx.fillRect(0, 0, this.config.width, this.config.height); return; } diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index ce6aa9cc3..456519fb4 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -202,11 +202,12 @@ function createRenderer() { }); } -it("hides source layers in gaps while rendering timeline annotations at the output time", async () => { +it("renders a timeline gap as solid black", async () => { const renderer = createRenderer(); const camera = { visible: true }; - const webcam = { visible: true }; - const captions = { visible: true }; + const output = createMockCanvas(); + output.width = 1920; + output.height = 1080; const annotations = vi.fn(); const renderOutput = vi.fn(async () => {}); Object.assign(renderer, { @@ -214,15 +215,17 @@ it("hides source layers in gaps while rendering timeline annotations at the outp videoContainer: {}, videoMaskGraphics: {}, cameraContainer: camera, - webcamRootContainer: webcam, - captionContainer: captions, + ensureExportCompositeCanvas: () => ({ canvas: output, context: output.context }), updateAnnotationLayer: annotations, renderOutput, }); await renderer.renderFrame(null, 0, 0, 33333, 1500000); - for (const layer of [camera, webcam, captions]) expect(layer.visible).toBe(false); - expect(annotations).toHaveBeenCalledWith(1500); - expect(renderOutput).toHaveBeenCalledWith(1500); + expect(camera.visible).toBe(false); + expect(output.context.fillStyle).toBe("#000000"); + expect(output.context.fillRect).toHaveBeenCalledWith(0, 0, 1920, 1080); + expect(annotations).not.toHaveBeenCalled(); + expect(renderOutput).not.toHaveBeenCalled(); + expect(renderer.getCanvas()).toBe(output); }); describe("ModernFrameRenderer Pixi lifecycle", () => { diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 9dee18985..24a6f86dd 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -3118,13 +3118,11 @@ export class FrameRenderer { this.currentVideoTime = timestamp / 1_000_000; this.cameraContainer.visible = videoFrame !== null; if (!videoFrame) { - if (this.backgroundForwardFrameSource || this.backgroundVideoElement) { - await this.syncBackgroundFrame(backgroundTimelineTimestamp / 1_000_000); - } - if (this.webcamRootContainer) this.webcamRootContainer.visible = false; - if (this.captionContainer) this.captionContainer.visible = false; - this.updateAnnotationLayer(backgroundTimelineTimestamp / 1000); - await this.renderOutput(backgroundTimelineTimestamp / 1000); + const output = this.ensureExportCompositeCanvas(); + if (!output) throw new Error("Failed to create gap frame canvas"); + output.context.fillStyle = "#000000"; + output.context.fillRect(0, 0, output.canvas.width, output.canvas.height); + this.outputCanvasOverride = output.canvas; return; } if (this.captionContainer) this.captionContainer.visible = true; From d6e6a3e9c7fca961395abbf885aa14acf865a106 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:17:45 +1000 Subject: [PATCH 05/10] Refine clip timing and recover safely from preview and export failures --- src/components/video-editor/SettingsPanel.tsx | 10 + src/components/video-editor/VideoEditor.tsx | 6 +- src/components/video-editor/VideoPlayback.tsx | 9 +- .../video-editor/audio/useAudioPreviewSync.ts | 5 + .../video-editor/captionTimeline.test.ts | 58 ++++ .../video-editor/captionTimeline.ts | 51 +++ .../video-editor/export/useExportSettings.ts | 13 +- .../video-editor/hooks/useCaptionCommands.ts | 22 +- .../hooks/useClipRegionCommands.ts | 12 +- .../hooks/useTimelineEditingController.ts | 2 +- .../hooks/useTimelineProjection.ts | 13 +- .../video-editor/layout/EditorExportMenu.tsx | 25 +- .../layout/EditorTimelinePanel.tsx | 39 ++- .../useTimelineKeyboardShortcuts.test.ts | 67 ++++ .../hooks/useTimelineKeyboardShortcuts.ts | 7 + src/components/video-editor/types.test.ts | 14 + src/components/video-editor/types.ts | 4 +- .../videoPlayback/clipPlayback.test.ts | 74 +++- .../videoPlayback/clipPlayback.ts | 36 +- .../videoPlayback/playbackRate.test.ts | 22 ++ .../videoPlayback/playbackRate.ts | 18 + src/lib/exporter/audioEncoder.test.ts | 13 + src/lib/exporter/frameRenderer.ts | 1 + src/lib/exporter/modernFrameRenderer.test.ts | 139 ++++---- src/lib/exporter/modernFrameRenderer.ts | 321 +----------------- .../modernVideoExporter.cancellation.test.ts | 62 ++++ .../modernVideoExporter.fallback.test.ts | 74 ++++ src/lib/exporter/modernVideoExporter.ts | 84 +++-- src/lib/exporter/offlineAudioProcessor.ts | 11 +- src/lib/exporter/streamingDecodePipeline.ts | 20 +- src/lib/exporter/streamingDecoder.test.ts | 33 +- 31 files changed, 793 insertions(+), 472 deletions(-) create mode 100644 src/components/video-editor/captionTimeline.test.ts create mode 100644 src/components/video-editor/captionTimeline.ts create mode 100644 src/components/video-editor/timeline/hooks/useTimelineKeyboardShortcuts.test.ts create mode 100644 src/components/video-editor/videoPlayback/playbackRate.test.ts create mode 100644 src/components/video-editor/videoPlayback/playbackRate.ts create mode 100644 src/lib/exporter/modernVideoExporter.cancellation.test.ts diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 586e09d2a..8b289e3f5 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -91,6 +91,7 @@ import { } from "./types"; import { fromCursorSwaySliderValue, toCursorSwaySliderValue } from "./videoPlayback/cursorSway"; import { isZeroPadding } from "./videoPlayback/layoutUtils"; +import { supportsPreviewPlaybackRate } from "./videoPlayback/playbackRate"; import { cursorSetAssets, getCursorStyleSizeMultiplier, @@ -3057,6 +3058,15 @@ export function SettingsPanel({ key={option.speed} type="button" onClick={() => onClipSpeedChange?.(option.speed)} + disabled={!supportsPreviewPlaybackRate(option.speed)} + title={ + !supportsPreviewPlaybackRate(option.speed) + ? tSettings( + "speed.unsupported", + "Not supported for preview on this device", + ) + : undefined + } className={cn( "h-auto w-full rounded-lg border px-0.5 py-2 text-center shadow-sm transition-all duration-200 ease-out cursor-pointer", isActive diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index e3f98e1a1..6d22666a1 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -94,7 +94,11 @@ export default function VideoEditor() { const effectiveShowCursor = sessionShowCursorOverride ?? showCursor; const headerLeftControlsPaddingClass = appPlatform === "darwin" ? "pl-[76px]" : ""; const { cursorTelemetrySourcePath, autoCaptions, autoCaptionSettings } = timeline; - const exportSettings = useExportSettings(initialEditorPreferences, autoCaptions); + const exportSettings = useExportSettings( + initialEditorPreferences, + autoCaptions, + timeline.clipRegions, + ); const { includeCaptionSidecar, mp4FrameRate, diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 33b1c6793..afdc56e7a 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -91,7 +91,7 @@ import { isAnnotationActiveAtTime, shouldClearSelectedAnnotation, } from "./videoPlayback/annotationVisibility"; -import { createClipPlayback } from "./videoPlayback/clipPlayback"; +import { createClipPlayback, findPreviewClipAtTimelineTime } from "./videoPlayback/clipPlayback"; import { DEFAULT_FOCUS } from "./videoPlayback/constants"; import { type CursorFollowCameraState, @@ -113,6 +113,7 @@ import { stepSpringValue, } from "./videoPlayback/motionSmoothing"; import { updateOverlayIndicator } from "./videoPlayback/overlayUtils"; +import { supportsPreviewPlaybackRate } from "./videoPlayback/playbackRate"; import { PreviewVideoSource } from "./videoPlayback/previewVideoSource"; import { getSceneEffectMetrics } from "./videoPlayback/sceneEffects"; import { @@ -446,7 +447,7 @@ const VideoPlayback = forwardRef( null, ); const currentTime = mapTimelineTimeToSourceTime(timelineTime * 1000, clipRegions) / 1000; - const isGap = !findClipAtTimelineTime(timelineTime * 1000, clipRegions); + const isGap = !findPreviewClipAtTimelineTime(timelineTime * 1000, clipRegions); const clipRegionsRef = useRef(clipRegions); const clipPlaybackRef = useRef | null>(null); const onPlaybackErrorRef = useRef(onError); @@ -1717,6 +1718,10 @@ const VideoPlayback = forwardRef( const targetPlaybackRate = findClipAtTimelineTime(timelineTime * 1000, clipRegions)?.speed ?? 1; + if (!supportsPreviewPlaybackRate(targetPlaybackRate)) { + webcamVideo.pause(); + return; + } enablePitchPreservingPlayback(webcamVideo); if (Math.abs(webcamVideo.playbackRate - targetPlaybackRate) > 0.001) { webcamVideo.playbackRate = targetPlaybackRate; diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index c1e3a2594..78c1b02cb 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -9,6 +9,7 @@ import { resolvePreviewMediaDuration, } from "@/lib/mediaTiming"; import type { AudioRegion } from "../types"; +import { supportsPreviewPlaybackRate } from "../videoPlayback/playbackRate"; import { getAudioResourceVersionKey, getVersionedAudioResourceUrl, @@ -395,6 +396,10 @@ export function useAudioPreviewSync({ } for (const audio of sourceAudioElementsRef.current.values()) { + if (!supportsPreviewPlaybackRate(sourcePlaybackRate)) { + audio.pause(); + continue; + } const sourceAudioPath = audio.dataset.sourceAudioPath ?? ""; audio.volume = Math.max( 0, diff --git a/src/components/video-editor/captionTimeline.test.ts b/src/components/video-editor/captionTimeline.test.ts new file mode 100644 index 000000000..6b77a4d27 --- /dev/null +++ b/src/components/video-editor/captionTimeline.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { captionSpanToSource, projectCaptionCues, retimeCaptionFragment } from "./captionTimeline"; + +const clips = [ + { id: "a", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 3 }, + { id: "b", startMs: 2000, endMs: 3000, sourceStartMs: 6000, speed: 3 }, +]; +describe("caption timeline projection", () => { + it("omits deleted captions and splits crossing cues around gaps with distinct identities", () => { + const fragments = projectCaptionCues( + [ + { id: "deleted", startMs: 3100, endMs: 5900, text: "gone" }, + { id: "crossing", startMs: 1500, endMs: 7500, text: "kept" }, + ], + clips, + ); + expect( + fragments.map(({ sourceCueId, startMs, endMs }) => ({ sourceCueId, startMs, endMs })), + ).toEqual([ + { sourceCueId: "crossing", startMs: 500, endMs: 1000 }, + { sourceCueId: "crossing", startMs: 2000, endMs: 2500 }, + ]); + expect(new Set(fragments.map(({ id }) => id)).size).toBe(2); + }); + it("projects reused footage at every timeline position", () => { + const cue = { id: "cue", startMs: 0, endMs: 1500, text: "hello" }; + const fragments = projectCaptionCues([cue], [clips[0], { ...clips[1], sourceStartMs: 0 }]); + expect(fragments.map(({ startMs, endMs }) => [startMs, endMs])).toEqual([ + [0, 500], + [2000, 2500], + ]); + expect(projectCaptionCues([cue], [])).toEqual([]); + }); + it("retimes within the selected clip even at its cut boundary", () => { + expect(captionSpanToSource(clips[0], { start: 500, end: 1000 })).toEqual({ + startMs: 1500, + endMs: 3000, + }); + expect(captionSpanToSource(clips[1], { start: 2000, end: 4000 })).toEqual({ + startMs: 6000, + endMs: 9000, + }); + }); + it("preserves the other fragment when resizing one edge of a crossing cue", () => { + const fragments = projectCaptionCues( + [{ id: "cue", startMs: 1500, endMs: 7500, text: "hello" }], + clips, + ); + expect(retimeCaptionFragment(fragments[0], { start: 600, end: 1000 })).toEqual({ + startMs: 1800, + endMs: 7500, + }); + expect(retimeCaptionFragment(fragments[1], { start: 2000, end: 2600 })).toEqual({ + startMs: 1500, + endMs: 7800, + }); + }); +}); diff --git a/src/components/video-editor/captionTimeline.ts b/src/components/video-editor/captionTimeline.ts new file mode 100644 index 000000000..3ba237f21 --- /dev/null +++ b/src/components/video-editor/captionTimeline.ts @@ -0,0 +1,51 @@ +import { + type CaptionCue, + type ClipRegion, + getClipSourceEndMs, + getClipSourceStartMs, + sortClipRegions, +} from "./types"; + +export function captionSpanToSource(clip: ClipRegion, span: { start: number; end: number }) { + const sourceTime = (time: number) => + Math.round( + getClipSourceStartMs(clip) + + (Math.max(clip.startMs, Math.min(clip.endMs, time)) - clip.startMs) * clip.speed, + ); + return { startMs: sourceTime(span.start), endMs: sourceTime(span.end) }; +} + +export function projectCaptionCues(cues: CaptionCue[], clips: ClipRegion[]) { + return sortClipRegions(clips).flatMap((clip) => { + const sourceStart = getClipSourceStartMs(clip); + const sourceEnd = getClipSourceEndMs(clip); + return cues.flatMap((cue) => { + const start = Math.max(cue.startMs, sourceStart); + const end = Math.min(cue.endMs, sourceEnd); + if (start >= end) return []; + return [ + { + ...cue, + id: JSON.stringify([cue.id, clip.id]), + sourceCueId: cue.id, + sourceCue: cue, + clip, + startMs: clip.startMs + (start - sourceStart) / clip.speed, + endMs: clip.startMs + (end - sourceStart) / clip.speed, + }, + ]; + }); + }); +} + +export function retimeCaptionFragment( + fragment: ReturnType[number], + span: { start: number; end: number }, +) { + const mapped = captionSpanToSource(fragment.clip, span); + // An unchanged clipped edge must not truncate the rest of the source cue. + return { + startMs: span.start === fragment.startMs ? fragment.sourceCue.startMs : mapped.startMs, + endMs: span.end === fragment.endMs ? fragment.sourceCue.endMs : mapped.endMs, + }; +} diff --git a/src/components/video-editor/export/useExportSettings.ts b/src/components/video-editor/export/useExportSettings.ts index 7cd88330b..1f3384d67 100644 --- a/src/components/video-editor/export/useExportSettings.ts +++ b/src/components/video-editor/export/useExportSettings.ts @@ -9,12 +9,17 @@ import type { GifFrameRate, GifSizePreset, } from "@/lib/exporter"; +import { projectCaptionCues } from "../captionTimeline"; import type { EditorPreferences } from "../editorPreferences"; -import type { CaptionCue } from "../types"; +import type { CaptionCue, ClipRegion } from "../types"; const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30; -export function useExportSettings(preferences: EditorPreferences, autoCaptions: CaptionCue[]) { +export function useExportSettings( + preferences: EditorPreferences, + autoCaptions: CaptionCue[], + clips: ClipRegion[], +) { const [includeCaptionSidecar, setIncludeCaptionSidecar] = useState(false); const [exportQuality, setExportQuality] = useState(preferences.exportQuality); const [exportEncodingMode, setExportEncodingMode] = useState( @@ -35,7 +40,7 @@ export function useExportSettings(preferences: EditorPreferences, autoCaptions: const [gifSizePreset, setGifSizePreset] = useState(preferences.gifSizePreset); const captionSidecarCues = useMemo( () => - autoCaptions + projectCaptionCues(autoCaptions, clips) .filter( (cue) => Number.isFinite(cue.startMs) && @@ -45,7 +50,7 @@ export function useExportSettings(preferences: EditorPreferences, autoCaptions: cue.text.trim().length > 0, ) .map(({ startMs, endMs, text }) => ({ startMs, endMs, text })), - [autoCaptions], + [autoCaptions, clips], ); return { diff --git a/src/components/video-editor/hooks/useCaptionCommands.ts b/src/components/video-editor/hooks/useCaptionCommands.ts index d541cbcbc..5326070db 100644 --- a/src/components/video-editor/hooks/useCaptionCommands.ts +++ b/src/components/video-editor/hooks/useCaptionCommands.ts @@ -15,10 +15,18 @@ import { retimeCue, splitCue, } from "../captionOps"; -import type { AutoCaptionSettings, CaptionCue, EditorEffectSection } from "../types"; +import { captionSpanToSource } from "../captionTimeline"; +import { + type AutoCaptionSettings, + type CaptionCue, + type ClipRegion, + type EditorEffectSection, + findClipAtTimelineTime, +} from "../types"; import type { VideoPlaybackRef } from "../VideoPlayback"; interface UseCaptionCommandsParams { + clipRegions: ClipRegion[]; autoCaptions: CaptionCue[]; setAutoCaptions: Dispatch>; setAutoCaptionSettings: Dispatch>; @@ -30,11 +38,11 @@ interface UseCaptionCommandsParams { setActiveEffectSection: Dispatch>; videoPlaybackRef: RefObject; mapSourceTimeToTimelineTime: (timeMs: number) => number; - mapTimelineTimeToSourceTime: (timeMs: number) => number; handleSeek: (time: number, options?: { pause?: boolean }) => void; } export function useCaptionCommands({ + clipRegions, autoCaptions, setAutoCaptions, setAutoCaptionSettings, @@ -46,7 +54,6 @@ export function useCaptionCommands({ setActiveEffectSection, videoPlaybackRef, mapSourceTimeToTimelineTime, - mapTimelineTimeToSourceTime, handleSeek, }: UseCaptionCommandsParams) { const handleSelectCaption = useCallback( @@ -157,11 +164,10 @@ export function useCaptionCommands({ const handleCaptionAdded = useCallback( (span: Span) => { + const clip = findClipAtTimelineTime(span.start, clipRegions); + if (!clip) return; cancelEdit(); - const newCue = createCaptionCue({ - startMs: mapTimelineTimeToSourceTime(span.start), - endMs: mapTimelineTimeToSourceTime(span.end), - }); + const newCue = createCaptionCue(captionSpanToSource(clip, span)); setAutoCaptions((captions) => addCue(captions, newCue)); setSelectedCaptionId(newCue.id); setActiveEffectSection("caption"); @@ -174,7 +180,7 @@ export function useCaptionCommands({ [ cancelEdit, handleSeek, - mapTimelineTimeToSourceTime, + clipRegions, setActiveEffectSection, setAutoCaptions, setSelectedAnnotationId, diff --git a/src/components/video-editor/hooks/useClipRegionCommands.ts b/src/components/video-editor/hooks/useClipRegionCommands.ts index 3470aa07b..f605789e9 100644 --- a/src/components/video-editor/hooks/useClipRegionCommands.ts +++ b/src/components/video-editor/hooks/useClipRegionCommands.ts @@ -1,10 +1,11 @@ import type { Span } from "dnd-timeline"; import { type Dispatch, type MutableRefObject, type SetStateAction, useCallback } from "react"; import { toast } from "sonner"; -import { planClipSpeedChange } from "../clipSpeedChange"; import { changeClipSpan } from "../clipSpanChange"; +import { planClipSpeedChange } from "../clipSpeedChange"; import { planClipSplit } from "../clipSplit"; import type { ClipRegion, EditorEffectSection, ZoomRegion } from "../types"; +import { supportsPreviewPlaybackRate } from "../videoPlayback/playbackRate"; type Translator = ( key: string, @@ -123,6 +124,15 @@ export function useClipRegionCommands({ const handleClipSpeedChange = useCallback( (speed: number) => { if (!selectedClipId || !Number.isFinite(speed) || speed <= 0) return; + if (!supportsPreviewPlaybackRate(speed)) { + toast.error( + t( + "editor.timeline.unsupportedSpeed", + "This speed is not supported for preview on this device.", + ), + ); + return; + } const plan = planClipSpeedChange({ clipRegions, zoomRegions, selectedClipId, speed }); if (!plan) return; if ("blockedReason" in plan) { diff --git a/src/components/video-editor/hooks/useTimelineEditingController.ts b/src/components/video-editor/hooks/useTimelineEditingController.ts index cc42f864e..083340f9a 100644 --- a/src/components/video-editor/hooks/useTimelineEditingController.ts +++ b/src/components/video-editor/hooks/useTimelineEditingController.ts @@ -109,6 +109,7 @@ export function useTimelineEditingController(input: Input) { timelineDuration: projection.timelineDuration, }); const captionCommands = useCaptionCommands({ + clipRegions: timeline.clipRegions, autoCaptions: timeline.autoCaptions, setAutoCaptions: timeline.setAutoCaptions, setAutoCaptionSettings: timeline.setAutoCaptionSettings, @@ -120,7 +121,6 @@ export function useTimelineEditingController(input: Input) { setActiveEffectSection: input.setActiveEffectSection, videoPlaybackRef: input.videoPlaybackRef, mapSourceTimeToTimelineTime: projection.mapSourceTimeToTimelineTime, - mapTimelineTimeToSourceTime: projection.mapTimelineTimeToSourceTime, handleSeek: playback.handleSeek, }); const zoomCommands = useZoomRegionCommands({ diff --git a/src/components/video-editor/hooks/useTimelineProjection.ts b/src/components/video-editor/hooks/useTimelineProjection.ts index a455c29ea..7d88ffa74 100644 --- a/src/components/video-editor/hooks/useTimelineProjection.ts +++ b/src/components/video-editor/hooks/useTimelineProjection.ts @@ -1,9 +1,9 @@ /* biome-ignore-all lint/correctness/useExhaustiveDependencies: mutable timeline bootstrap refs intentionally do not trigger effects. */ import { type MutableRefObject, useCallback, useEffect, useMemo } from "react"; +import { projectCaptionCues } from "../captionTimeline"; import { deriveNextId } from "../projectPersistence"; import type { useTimelineState } from "../state/useTimelineState"; import { - type CaptionCue, clipsToTrims, extendAutoFullTrackClip, getClipSourceEndMs, @@ -90,14 +90,9 @@ export function useTimelineProjection({ [clipRegions], ); const effectiveZoomRegions: ZoomRegion[] = zoomRegions; - const effectiveCaptionRegions = useMemo( - () => - autoCaptions.map((cue) => ({ - ...cue, - startMs: toTimelineTime(cue.startMs), - endMs: toTimelineTime(cue.endMs), - })), - [autoCaptions, toTimelineTime], + const effectiveCaptionRegions = useMemo( + () => projectCaptionCues(autoCaptions, clipRegions), + [autoCaptions, clipRegions], ); const timelinePlayheadTime = currentTime; const timelineDuration = useMemo( diff --git a/src/components/video-editor/layout/EditorExportMenu.tsx b/src/components/video-editor/layout/EditorExportMenu.tsx index b39a8521e..7f557a70a 100644 --- a/src/components/video-editor/layout/EditorExportMenu.tsx +++ b/src/components/video-editor/layout/EditorExportMenu.tsx @@ -1,4 +1,5 @@ import { DownloadSimple as Download } from "@phosphor-icons/react"; +import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -190,10 +191,32 @@ export function EditorExportMenu(props: Props) { Path: {exportRuntimeLabel}

) : null} -

+

{exportError}

+ {hasPendingExportSave ? (