From 768e032eb93f0500f95a6a673872e9b9d2a1e445 Mon Sep 17 00:00:00 2001 From: Nikhil Sonti Date: Fri, 24 Jul 2026 14:27:40 -0700 Subject: [PATCH 1/2] fix(replay): auto-follow tool activity across tabs Sequential tab chapters (#2024) made the visible tab's rrweb stream the transport, so any tool that ran after its tab stopped emitting DOM events became unreachable. In the 2026-07-24 Trendshift session tab 1 records 9.003s of events but receives tools at 24.225s, 26.717s and 38.321s; the chapter ended at 9.003s and only 6 of 16 tab-attributed tools fell inside their tab's span at all. Untabbed tools such as name_session were filtered out of every chapter. Replaces chapter orchestration with one global activity clock and a pure session plan: - session-replay.ts derives the current tool, the visible tab, and the tab-local projection as a function of global time. No camera history, so a backward seek, a forward seek, and a live refresh all agree. - usePlayback owns activity time from animation-frame timestamps instead of mirroring rrweb, and resumes when a live recording grows past where it stopped. - ReplayViewport keeps its single Replayer and aligns it to the projected time, holding a boundary frame while global activity continues past the end of the track. Caption and address bar now have separate sources. - The action timeline is global again: every tool is listed, untabbed and no-visual tools stay current without moving the camera, and a tab click pins the camera only. Tool starts come from the durationMs already on the dispatch row; missing, negative, and non-finite values fall back to completion. --- .../apps/claw-app/modules/api/replay.hooks.ts | 10 +- .../claw-app/screens/replay/EventTimeline.tsx | 42 +- .../screens/replay/PlaybackTransport.test.tsx | 60 +- .../screens/replay/PlaybackTransport.tsx | 26 +- .../claw-app/screens/replay/Replay.test.tsx | 1686 +++++------------ .../apps/claw-app/screens/replay/Replay.tsx | 764 ++------ .../screens/replay/ReplayViewport.test.tsx | 306 ++- .../screens/replay/ReplayViewport.tsx | 153 +- .../screens/replay/replay.data.test.ts | 46 +- .../claw-app/screens/replay/replay.data.ts | 8 +- .../claw-app/screens/replay/replay.helpers.ts | 18 +- .../screens/replay/session-replay.test.ts | 402 ++++ .../claw-app/screens/replay/session-replay.ts | 285 +++ .../claw-app/screens/replay/tab-view.test.ts | 118 +- .../apps/claw-app/screens/replay/tab-view.ts | 69 +- .../screens/replay/use-playback.test.tsx | 243 +++ .../claw-app/screens/replay/use-playback.ts | 136 +- 17 files changed, 2192 insertions(+), 2180 deletions(-) create mode 100644 packages/browseros-agent/apps/claw-app/screens/replay/session-replay.test.ts create mode 100644 packages/browseros-agent/apps/claw-app/screens/replay/session-replay.ts create mode 100644 packages/browseros-agent/apps/claw-app/screens/replay/use-playback.test.tsx diff --git a/packages/browseros-agent/apps/claw-app/modules/api/replay.hooks.ts b/packages/browseros-agent/apps/claw-app/modules/api/replay.hooks.ts index abd9d68cc1..6002b66c10 100644 --- a/packages/browseros-agent/apps/claw-app/modules/api/replay.hooks.ts +++ b/packages/browseros-agent/apps/claw-app/modules/api/replay.hooks.ts @@ -25,8 +25,16 @@ export type ReplayVerb = export type ReplayKind = 'action' | 'block' | 'done' export interface ReplayFrame { - /** Seconds into the session. */ + /** Seconds into the session, at which this dispatch completed. */ t: number + /** + * How long the tool ran, from the source dispatch row. `session-replay.ts` + * subtracts it from `t` to approximate when the action began, which is when + * the replay starts treating it as current. Absent on rows recorded before + * durations were persisted; negative and non-finite values fall back to the + * completion time. + */ + durationMs?: number | null kind: ReplayKind verb: ReplayVerb /** Short node label, e.g. the page title or a focused element. */ diff --git a/packages/browseros-agent/apps/claw-app/screens/replay/EventTimeline.tsx b/packages/browseros-agent/apps/claw-app/screens/replay/EventTimeline.tsx index 5239cf85d6..901ee7555f 100644 --- a/packages/browseros-agent/apps/claw-app/screens/replay/EventTimeline.tsx +++ b/packages/browseros-agent/apps/claw-app/screens/replay/EventTimeline.tsx @@ -1,24 +1,26 @@ import { Layers } from 'lucide-react' import { cn } from '@/lib/utils' -import type { ReplayFrame } from '@/modules/api/replay.hooks' import { formatTime, KIND_STYLE, VERB_META } from './replay.helpers' +import type { ReplayAction } from './session-replay' interface EventTimelineProps { - frames: readonly ReplayFrame[] - currentFrameIndex: number - onSelectFrame: (frame: ReplayFrame) => void + /** Every tool in the session, globally ordered — not filtered by tab. */ + actions: readonly ReplayAction[] + /** Index of the globally current action; -1 before the first one starts. */ + currentIndex: number + onSelectAction: (action: ReplayAction) => void } /** - * Right-rail vertical event list. Each row corresponds to one frame - * in the recording. Past frames are full-opacity, future frames dim, - * and the row matching the playhead gets an accent-tint background. - * Clicking a row seeks the player to that frame. + * Right-rail vertical list of the session's tools in chronological order. + * Rows are stamped with the action's activity start, which is both where the + * row becomes current and where clicking it seeks. Past rows are full-opacity, + * future rows dim, and the current row gets an accent-tint background. */ export function EventTimeline({ - frames, - currentFrameIndex, - onSelectFrame, + actions, + currentIndex, + onSelectAction, }: EventTimelineProps) { return (