From 7e1f19b01f96be583b31473a8036cc46dc997216 Mon Sep 17 00:00:00 2001 From: soukiassianb Date: Tue, 28 Jul 2026 21:03:57 +0200 Subject: [PATCH 1/3] feat: report tool calls on $ai_generation Tool calls were only emitted as $ai_span events. PostHog's ingestion extracts tool usage exclusively from $ai_generation, so the AI observability Tools view was empty for every OpenCode user despite the tool data being captured. Each generation now carries $ai_tools_called with the names of the tools that step called, in call order. Names are accumulated between step-start and step-finish, the same window the existing span parenting relies on. Spans are unchanged. --- .changeset/report-tools-called.md | 5 ++ src/events.test.ts | 28 ++++++++++ src/events.ts | 8 +++ src/index.test.ts | 89 +++++++++++++++++++++++++++++++ src/index.ts | 9 ++++ src/types.ts | 2 + 6 files changed, 141 insertions(+) create mode 100644 .changeset/report-tools-called.md diff --git a/.changeset/report-tools-called.md b/.changeset/report-tools-called.md new file mode 100644 index 0000000..e07adf5 --- /dev/null +++ b/.changeset/report-tools-called.md @@ -0,0 +1,5 @@ +--- +'@posthog/opencode': patch +--- + +Report tool calls on `$ai_generation` so PostHog's AI observability Tools view is populated. Tool calls were only emitted as `$ai_span` events, but PostHog extracts tool usage exclusively from `$ai_generation` — so the Tools tab, Tool trends, and Tool co-occurrence stayed empty for every OpenCode user. Each generation now carries `$ai_tools_called` with the names of the tools that step called, in call order. Spans are unchanged, so the trace timeline keeps its existing shape. diff --git a/src/events.test.ts b/src/events.test.ts index 6f7a9f2..c83ef61 100644 --- a/src/events.test.ts +++ b/src/events.test.ts @@ -58,6 +58,7 @@ function makeTrace(overrides?: Partial): TraceState { stepInputMessages: [{ role: 'user', content: 'Hello' }], stepInputSnapshot: [{ role: 'user', content: 'Hello' }], stepAssistantText: 'Hi there!', + stepToolCalls: [], messageIds: new Set(), ...overrides, } @@ -167,6 +168,33 @@ describe('buildAiGeneration', () => { expect(result.properties.$ai_output_choices).toBeNull() }) + it('reports the tools called during the step, in call order', () => { + const trace = makeTrace({ stepToolCalls: ['read', 'edit', 'read'] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) + expect(result.properties.$ai_tools_called).toEqual(['read', 'edit', 'read']) + }) + + it('reports no tools called when the step called none', () => { + const trace = makeTrace({ stepToolCalls: [] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) + expect(result.properties.$ai_tools_called).toBeNull() + }) + + it('still reports tool names in privacy mode', () => { + // Names carry no user content — $ai_span_name already sends them unredacted. + const trace = makeTrace({ stepToolCalls: ['bash'] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, privacyConfig) + expect(result.properties.$ai_tools_called).toEqual(['bash']) + expect(result.properties.$ai_output_choices).toBeNull() + }) + + it('copies the tool names so later steps cannot mutate a captured event', () => { + const trace = makeTrace({ stepToolCalls: ['read'] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) + trace.stepToolCalls.push('edit') + expect(result.properties.$ai_tools_called).toEqual(['read']) + }) + it('marks error generations', () => { const assistant = makeAssistantInfo({ error: { name: 'UnknownError', data: { message: 'Rate limited' } }, diff --git a/src/events.ts b/src/events.ts index bbb2e10..dabbec7 100644 --- a/src/events.ts +++ b/src/events.ts @@ -66,6 +66,14 @@ export function buildAiGeneration( $ai_input: inputMessages, $ai_output_choices: outputChoices, + // Tool calls are emitted as $ai_span events too, but PostHog only extracts + // tool usage from $ai_generation, so the Tools view stays empty unless the + // names are reported here as well. A user-provided value is respected and + // normalized server-side, and $ai_tool_call_count is derived from it. + // Names only — no arguments — so this carries no more than $ai_span_name, + // which is already sent unredacted in privacy mode. + $ai_tools_called: trace.stepToolCalls.length > 0 ? [...trace.stepToolCalls] : null, + $ai_is_error: !!assistantInfo?.error, $ai_error: serializeError(assistantInfo?.error, config.maxAttributeLength), diff --git a/src/index.test.ts b/src/index.test.ts index e7accae..7ac5d4f 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -104,4 +104,93 @@ describe('trace state machine (real OpenCode event ordering)', () => { // exactly one trace per user message (no duplicate traces from repeats) expect(captured.filter((e) => e.event === '$ai_trace')).toHaveLength(1) }) + + it("attributes each step's tool calls to that step's generation", async () => { + // Two steps: the first calls a tool, the second answers from its result. + // Tool parts arrive between step-start and step-finish, which is the same + // ordering the existing span parenting already relies on. + const tool = (name: string, id: string) => ({ + type: 'message.part.updated', + properties: { + part: { + type: 'tool', + id, + sessionID: S, + messageID: A, + tool: name, + state: { + status: 'completed', + input: { path: 'a.txt' }, + output: 'file contents', + time: { start: 0, end: 1000 }, + }, + }, + }, + }) + const stepFinish = () => ({ + type: 'message.part.updated', + properties: { + part: { + type: 'step-finish', + sessionID: S, + messageID: A, + tokens: { input: 10, output: 5, reasoning: 0, cache: { read: 0, write: 0 } }, + cost: 0, + reason: 'tool-calls', + }, + }, + }) + + await run([ + { + type: 'message.updated', + properties: { info: { role: 'user', id: U, sessionID: S, time: { created: 1 }, agent: 'build' } }, + }, + { + type: 'message.part.updated', + properties: { part: { type: 'text', messageID: U, sessionID: S, text: 'read a.txt' } }, + }, + { + type: 'message.updated', + properties: { + info: { + role: 'assistant', + id: A, + sessionID: S, + modelID: 'gpt-5.4-mini', + providerID: 'openai', + time: { created: 1 }, + }, + }, + }, + // step 1 — calls two tools + { type: 'message.part.updated', properties: { part: { type: 'step-start', sessionID: S, messageID: A } } }, + tool('read', 'part-tool-1'), + tool('edit', 'part-tool-2'), + stepFinish(), + // step 2 — pure text answer, no tools + { type: 'message.part.updated', properties: { part: { type: 'step-start', sessionID: S, messageID: A } } }, + { + type: 'message.part.updated', + properties: { part: { type: 'text', messageID: A, sessionID: S, text: 'Done.' } }, + }, + stepFinish(), + { type: 'session.idle', properties: { sessionID: S } }, + ]) + + const generations = captured.filter((e) => e.event === '$ai_generation') + expect(generations).toHaveLength(2) + + // Step 1 reports both tools, in call order; step 2 reports none. + expect(generations[0].properties.$ai_tools_called).toEqual(['read', 'edit']) + expect(generations[1].properties.$ai_tools_called).toBeNull() + + // The tool spans are parented to the same generation the names were + // attributed to, so both views of the step agree. + const spans = captured.filter((e) => e.event === '$ai_span') + expect(spans.map((s) => s.properties.$ai_span_name)).toEqual(['read', 'edit']) + for (const span of spans) { + expect(span.properties.$ai_parent_id).toBe(generations[0].properties.$ai_span_id) + } + }) }) diff --git a/src/index.ts b/src/index.ts index 2d38d60..ec6f007 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,7 @@ export const PostHogPlugin: Plugin = async () => { hadError: false, stepInputMessages: [], stepInputSnapshot: [], + stepToolCalls: [], messageIds: new Set(), } traces.set(sessionId, trace) @@ -90,6 +91,7 @@ export const PostHogPlugin: Plugin = async () => { agentName: msg.agent, stepInputMessages: [], stepInputSnapshot: [], + stepToolCalls: [], messageIds: new Set([msg.id]), } traces.set(msg.sessionID, trace) @@ -165,6 +167,9 @@ export const PostHogPlugin: Plugin = async () => { trace.stepInputSnapshot = [...trace.stepInputMessages] // Reset per-step assistant text for the new generation trace.stepAssistantText = undefined + // Reset per-step tool calls; they are attributed to the generation + // whose span ID was just allocated above. + trace.stepToolCalls = [] } function handleStepFinish(part: StepFinishPart) { @@ -195,6 +200,10 @@ export const PostHogPlugin: Plugin = async () => { const span = buildAiSpan(part.tool, toolState, trace, config) safeCapture(span) + // Also record the name on the step, so the generation can report which + // tools it called. The span above is parented to the same generation. + trace.stepToolCalls.push(part.tool) + // Feed tool result into step input so subsequent generations include // the tool context the model actually saw. Redact and truncate to // match the treatment applied to $ai_span fields. diff --git a/src/types.ts b/src/types.ts index 0d9d611..253b9cf 100644 --- a/src/types.ts +++ b/src/types.ts @@ -31,6 +31,8 @@ export interface TraceState { stepInputSnapshot: InputMessage[] /** Assistant text accumulated during the current step, reset on each step-start. */ stepAssistantText?: string + /** Names of tools called during the current step, in call order, reset on each step-start. */ + stepToolCalls: string[] currentAssistantMsg?: AssistantInfo currentGenerationSpanId?: string agentName?: string From 3464ef0d2a45c083bdfc0f922506d584d8a08636 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Tue, 11 Aug 2026 10:08:04 +0200 Subject: [PATCH 2/3] fix: address tool call review feedback --- src/events.test.ts | 7 ++++--- src/events.ts | 5 ++--- src/index.test.ts | 46 +++++++++++++++++++++++++++------------------- src/index.ts | 18 ++++++++++++------ src/types.ts | 2 ++ 5 files changed, 47 insertions(+), 31 deletions(-) diff --git a/src/events.test.ts b/src/events.test.ts index c83ef61..12825d6 100644 --- a/src/events.test.ts +++ b/src/events.test.ts @@ -59,6 +59,7 @@ function makeTrace(overrides?: Partial): TraceState { stepInputSnapshot: [{ role: 'user', content: 'Hello' }], stepAssistantText: 'Hi there!', stepToolCalls: [], + stepToolCallIds: new Set(), messageIds: new Set(), ...overrides, } @@ -171,7 +172,7 @@ describe('buildAiGeneration', () => { it('reports the tools called during the step, in call order', () => { const trace = makeTrace({ stepToolCalls: ['read', 'edit', 'read'] }) const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) - expect(result.properties.$ai_tools_called).toEqual(['read', 'edit', 'read']) + expect(result.properties.$ai_tools_called).toBe('read,edit,read') }) it('reports no tools called when the step called none', () => { @@ -184,7 +185,7 @@ describe('buildAiGeneration', () => { // Names carry no user content — $ai_span_name already sends them unredacted. const trace = makeTrace({ stepToolCalls: ['bash'] }) const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, privacyConfig) - expect(result.properties.$ai_tools_called).toEqual(['bash']) + expect(result.properties.$ai_tools_called).toBe('bash') expect(result.properties.$ai_output_choices).toBeNull() }) @@ -192,7 +193,7 @@ describe('buildAiGeneration', () => { const trace = makeTrace({ stepToolCalls: ['read'] }) const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) trace.stepToolCalls.push('edit') - expect(result.properties.$ai_tools_called).toEqual(['read']) + expect(result.properties.$ai_tools_called).toBe('read') }) it('marks error generations', () => { diff --git a/src/events.ts b/src/events.ts index dabbec7..c535889 100644 --- a/src/events.ts +++ b/src/events.ts @@ -68,11 +68,10 @@ export function buildAiGeneration( // Tool calls are emitted as $ai_span events too, but PostHog only extracts // tool usage from $ai_generation, so the Tools view stays empty unless the - // names are reported here as well. A user-provided value is respected and - // normalized server-side, and $ai_tool_call_count is derived from it. + // names are reported here as a canonical comma-separated string. // Names only — no arguments — so this carries no more than $ai_span_name, // which is already sent unredacted in privacy mode. - $ai_tools_called: trace.stepToolCalls.length > 0 ? [...trace.stepToolCalls] : null, + $ai_tools_called: trace.stepToolCalls.length > 0 ? trace.stepToolCalls.join(',') : null, $ai_is_error: !!assistantInfo?.error, $ai_error: serializeError(assistantInfo?.error, config.maxAttributeLength), diff --git a/src/index.test.ts b/src/index.test.ts index 7ac5d4f..15f1276 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -105,25 +105,31 @@ describe('trace state machine (real OpenCode event ordering)', () => { expect(captured.filter((e) => e.event === '$ai_trace')).toHaveLength(1) }) - it("attributes each step's tool calls to that step's generation", async () => { - // Two steps: the first calls a tool, the second answers from its result. - // Tool parts arrive between step-start and step-finish, which is the same - // ordering the existing span parenting already relies on. - const tool = (name: string, id: string) => ({ + it("attributes each step's tool calls in invocation order when parallel tools finish out of order", async () => { + // Two steps: the first calls parallel tools, the second answers from their results. + const tool = (name: string, id: string, status: 'running' | 'completed') => ({ type: 'message.part.updated', properties: { part: { type: 'tool', id, + callID: id, sessionID: S, messageID: A, tool: name, - state: { - status: 'completed', - input: { path: 'a.txt' }, - output: 'file contents', - time: { start: 0, end: 1000 }, - }, + state: + status === 'running' + ? { + status, + input: { path: 'a.txt' }, + time: { start: 0 }, + } + : { + status, + input: { path: 'a.txt' }, + output: 'file contents', + time: { start: 0, end: 1000 }, + }, }, }, }) @@ -163,10 +169,12 @@ describe('trace state machine (real OpenCode event ordering)', () => { }, }, }, - // step 1 — calls two tools + // step 1 — invokes read before edit, but edit completes first { type: 'message.part.updated', properties: { part: { type: 'step-start', sessionID: S, messageID: A } } }, - tool('read', 'part-tool-1'), - tool('edit', 'part-tool-2'), + tool('read', 'part-tool-1', 'running'), + tool('edit', 'part-tool-2', 'running'), + tool('edit', 'part-tool-2', 'completed'), + tool('read', 'part-tool-1', 'completed'), stepFinish(), // step 2 — pure text answer, no tools { type: 'message.part.updated', properties: { part: { type: 'step-start', sessionID: S, messageID: A } } }, @@ -181,14 +189,14 @@ describe('trace state machine (real OpenCode event ordering)', () => { const generations = captured.filter((e) => e.event === '$ai_generation') expect(generations).toHaveLength(2) - // Step 1 reports both tools, in call order; step 2 reports none. - expect(generations[0].properties.$ai_tools_called).toEqual(['read', 'edit']) + // Step 1 reports both tools in invocation order, as a canonical string; + // step 2 reports none. + expect(generations[0].properties.$ai_tools_called).toBe('read,edit') expect(generations[1].properties.$ai_tools_called).toBeNull() - // The tool spans are parented to the same generation the names were - // attributed to, so both views of the step agree. + // Spans still arrive in completion order and are parented to the same generation. const spans = captured.filter((e) => e.event === '$ai_span') - expect(spans.map((s) => s.properties.$ai_span_name)).toEqual(['read', 'edit']) + expect(spans.map((s) => s.properties.$ai_span_name)).toEqual(['edit', 'read']) for (const span of spans) { expect(span.properties.$ai_parent_id).toBe(generations[0].properties.$ai_span_id) } diff --git a/src/index.ts b/src/index.ts index ec6f007..db3829d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,6 +60,7 @@ export const PostHogPlugin: Plugin = async () => { stepInputMessages: [], stepInputSnapshot: [], stepToolCalls: [], + stepToolCallIds: new Set(), messageIds: new Set(), } traces.set(sessionId, trace) @@ -92,6 +93,7 @@ export const PostHogPlugin: Plugin = async () => { stepInputMessages: [], stepInputSnapshot: [], stepToolCalls: [], + stepToolCallIds: new Set(), messageIds: new Set([msg.id]), } traces.set(msg.sessionID, trace) @@ -170,6 +172,7 @@ export const PostHogPlugin: Plugin = async () => { // Reset per-step tool calls; they are attributed to the generation // whose span ID was just allocated above. trace.stepToolCalls = [] + trace.stepToolCallIds = new Set() } function handleStepFinish(part: StepFinishPart) { @@ -191,19 +194,22 @@ export const PostHogPlugin: Plugin = async () => { } function handleToolPart(part: ToolPart) { - if (part.state.status !== 'completed' && part.state.status !== 'error') return - const trace = traces.get(part.sessionID) if (!trace) return + // Record invocation order as tools start. Terminal updates can arrive in + // a different order for parallel calls, and each call emits multiple updates. + if (part.state.status !== 'pending' && !trace.stepToolCallIds.has(part.callID)) { + trace.stepToolCallIds.add(part.callID) + trace.stepToolCalls.push(part.tool) + } + + if (part.state.status !== 'completed' && part.state.status !== 'error') return + const toolState = part.state as ToolStateCompleted | ToolStateError const span = buildAiSpan(part.tool, toolState, trace, config) safeCapture(span) - // Also record the name on the step, so the generation can report which - // tools it called. The span above is parented to the same generation. - trace.stepToolCalls.push(part.tool) - // Feed tool result into step input so subsequent generations include // the tool context the model actually saw. Redact and truncate to // match the treatment applied to $ai_span fields. diff --git a/src/types.ts b/src/types.ts index 253b9cf..0a1bef0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,6 +33,8 @@ export interface TraceState { stepAssistantText?: string /** Names of tools called during the current step, in call order, reset on each step-start. */ stepToolCalls: string[] + /** Tool call IDs already recorded for the current step. */ + stepToolCallIds: Set currentAssistantMsg?: AssistantInfo currentGenerationSpanId?: string agentName?: string From 275b36c4164903dc7230ae3666c9d2e46fb7c813 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Tue, 11 Aug 2026 10:19:37 +0200 Subject: [PATCH 3/3] test: document sequential event processing --- src/index.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.test.ts b/src/index.test.ts index 15f1276..f2e5396 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -20,6 +20,7 @@ type AnyEvent = { type: string; properties: Record } async function run(events: AnyEvent[]) { process.env.POSTHOG_API_KEY = 'phc_test' const hooks = (await PostHogPlugin({} as never)) as { event: (i: { event: AnyEvent }) => Promise } + // oxlint-disable-next-line no-await-in-loop -- Event order is significant. for (const event of events) await hooks.event({ event }) }