diff --git a/.changeset/report-tools-called.md b/.changeset/report-tools-called.md new file mode 100644 index 0000000..e07adf5 --- /dev/null +++ b/.changeset/report-tools-called.md @@ -0,0 +1,5 @@ +--- +'@posthog/opencode': patch +--- + +Report tool calls on `$ai_generation` so PostHog's AI observability Tools view is populated. Tool calls were only emitted as `$ai_span` events, but PostHog extracts tool usage exclusively from `$ai_generation` — so the Tools tab, Tool trends, and Tool co-occurrence stayed empty for every OpenCode user. Each generation now carries `$ai_tools_called` with the names of the tools that step called, in call order. Spans are unchanged, so the trace timeline keeps its existing shape. diff --git a/src/events.test.ts b/src/events.test.ts index 6f7a9f2..12825d6 100644 --- a/src/events.test.ts +++ b/src/events.test.ts @@ -58,6 +58,8 @@ function makeTrace(overrides?: Partial): TraceState { stepInputMessages: [{ role: 'user', content: 'Hello' }], stepInputSnapshot: [{ role: 'user', content: 'Hello' }], stepAssistantText: 'Hi there!', + stepToolCalls: [], + stepToolCallIds: new Set(), messageIds: new Set(), ...overrides, } @@ -167,6 +169,33 @@ describe('buildAiGeneration', () => { expect(result.properties.$ai_output_choices).toBeNull() }) + it('reports the tools called during the step, in call order', () => { + const trace = makeTrace({ stepToolCalls: ['read', 'edit', 'read'] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) + expect(result.properties.$ai_tools_called).toBe('read,edit,read') + }) + + it('reports no tools called when the step called none', () => { + const trace = makeTrace({ stepToolCalls: [] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) + expect(result.properties.$ai_tools_called).toBeNull() + }) + + it('still reports tool names in privacy mode', () => { + // Names carry no user content — $ai_span_name already sends them unredacted. + const trace = makeTrace({ stepToolCalls: ['bash'] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, privacyConfig) + expect(result.properties.$ai_tools_called).toBe('bash') + expect(result.properties.$ai_output_choices).toBeNull() + }) + + it('copies the tool names so later steps cannot mutate a captured event', () => { + const trace = makeTrace({ stepToolCalls: ['read'] }) + const result = buildAiGeneration(makeStepFinish(), makeAssistantInfo(), trace, defaultConfig) + trace.stepToolCalls.push('edit') + expect(result.properties.$ai_tools_called).toBe('read') + }) + it('marks error generations', () => { const assistant = makeAssistantInfo({ error: { name: 'UnknownError', data: { message: 'Rate limited' } }, diff --git a/src/events.ts b/src/events.ts index bbb2e10..c535889 100644 --- a/src/events.ts +++ b/src/events.ts @@ -66,6 +66,13 @@ export function buildAiGeneration( $ai_input: inputMessages, $ai_output_choices: outputChoices, + // Tool calls are emitted as $ai_span events too, but PostHog only extracts + // tool usage from $ai_generation, so the Tools view stays empty unless the + // names are reported here as a canonical comma-separated string. + // Names only — no arguments — so this carries no more than $ai_span_name, + // which is already sent unredacted in privacy mode. + $ai_tools_called: trace.stepToolCalls.length > 0 ? trace.stepToolCalls.join(',') : null, + $ai_is_error: !!assistantInfo?.error, $ai_error: serializeError(assistantInfo?.error, config.maxAttributeLength), diff --git a/src/index.test.ts b/src/index.test.ts index e7accae..f2e5396 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -20,6 +20,7 @@ type AnyEvent = { type: string; properties: Record } async function run(events: AnyEvent[]) { process.env.POSTHOG_API_KEY = 'phc_test' const hooks = (await PostHogPlugin({} as never)) as { event: (i: { event: AnyEvent }) => Promise } + // oxlint-disable-next-line no-await-in-loop -- Event order is significant. for (const event of events) await hooks.event({ event }) } @@ -104,4 +105,101 @@ describe('trace state machine (real OpenCode event ordering)', () => { // exactly one trace per user message (no duplicate traces from repeats) expect(captured.filter((e) => e.event === '$ai_trace')).toHaveLength(1) }) + + it("attributes each step's tool calls in invocation order when parallel tools finish out of order", async () => { + // Two steps: the first calls parallel tools, the second answers from their results. + const tool = (name: string, id: string, status: 'running' | 'completed') => ({ + type: 'message.part.updated', + properties: { + part: { + type: 'tool', + id, + callID: id, + sessionID: S, + messageID: A, + tool: name, + state: + status === 'running' + ? { + status, + input: { path: 'a.txt' }, + time: { start: 0 }, + } + : { + status, + input: { path: 'a.txt' }, + output: 'file contents', + time: { start: 0, end: 1000 }, + }, + }, + }, + }) + const stepFinish = () => ({ + type: 'message.part.updated', + properties: { + part: { + type: 'step-finish', + sessionID: S, + messageID: A, + tokens: { input: 10, output: 5, reasoning: 0, cache: { read: 0, write: 0 } }, + cost: 0, + reason: 'tool-calls', + }, + }, + }) + + await run([ + { + type: 'message.updated', + properties: { info: { role: 'user', id: U, sessionID: S, time: { created: 1 }, agent: 'build' } }, + }, + { + type: 'message.part.updated', + properties: { part: { type: 'text', messageID: U, sessionID: S, text: 'read a.txt' } }, + }, + { + type: 'message.updated', + properties: { + info: { + role: 'assistant', + id: A, + sessionID: S, + modelID: 'gpt-5.4-mini', + providerID: 'openai', + time: { created: 1 }, + }, + }, + }, + // step 1 — invokes read before edit, but edit completes first + { type: 'message.part.updated', properties: { part: { type: 'step-start', sessionID: S, messageID: A } } }, + tool('read', 'part-tool-1', 'running'), + tool('edit', 'part-tool-2', 'running'), + tool('edit', 'part-tool-2', 'completed'), + tool('read', 'part-tool-1', 'completed'), + stepFinish(), + // step 2 — pure text answer, no tools + { type: 'message.part.updated', properties: { part: { type: 'step-start', sessionID: S, messageID: A } } }, + { + type: 'message.part.updated', + properties: { part: { type: 'text', messageID: A, sessionID: S, text: 'Done.' } }, + }, + stepFinish(), + { type: 'session.idle', properties: { sessionID: S } }, + ]) + + const generations = captured.filter((e) => e.event === '$ai_generation') + expect(generations).toHaveLength(2) + + // Step 1 reports both tools in invocation order, as a canonical string; + // step 2 reports none. + expect(generations[0].properties.$ai_tools_called).toBe('read,edit') + expect(generations[1].properties.$ai_tools_called).toBeNull() + + // Spans still arrive in completion order and are parented to the same generation. + const spans = captured.filter((e) => e.event === '$ai_span') + expect(spans.map((s) => s.properties.$ai_span_name)).toEqual(['edit', 'read']) + for (const span of spans) { + expect(span.properties.$ai_parent_id).toBe(generations[0].properties.$ai_span_id) + } + }) }) diff --git a/src/index.ts b/src/index.ts index 2d38d60..db3829d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,8 @@ export const PostHogPlugin: Plugin = async () => { hadError: false, stepInputMessages: [], stepInputSnapshot: [], + stepToolCalls: [], + stepToolCallIds: new Set(), messageIds: new Set(), } traces.set(sessionId, trace) @@ -90,6 +92,8 @@ export const PostHogPlugin: Plugin = async () => { agentName: msg.agent, stepInputMessages: [], stepInputSnapshot: [], + stepToolCalls: [], + stepToolCallIds: new Set(), messageIds: new Set([msg.id]), } traces.set(msg.sessionID, trace) @@ -165,6 +169,10 @@ export const PostHogPlugin: Plugin = async () => { trace.stepInputSnapshot = [...trace.stepInputMessages] // Reset per-step assistant text for the new generation trace.stepAssistantText = undefined + // Reset per-step tool calls; they are attributed to the generation + // whose span ID was just allocated above. + trace.stepToolCalls = [] + trace.stepToolCallIds = new Set() } function handleStepFinish(part: StepFinishPart) { @@ -186,11 +194,18 @@ export const PostHogPlugin: Plugin = async () => { } function handleToolPart(part: ToolPart) { - if (part.state.status !== 'completed' && part.state.status !== 'error') return - const trace = traces.get(part.sessionID) if (!trace) return + // Record invocation order as tools start. Terminal updates can arrive in + // a different order for parallel calls, and each call emits multiple updates. + if (part.state.status !== 'pending' && !trace.stepToolCallIds.has(part.callID)) { + trace.stepToolCallIds.add(part.callID) + trace.stepToolCalls.push(part.tool) + } + + if (part.state.status !== 'completed' && part.state.status !== 'error') return + const toolState = part.state as ToolStateCompleted | ToolStateError const span = buildAiSpan(part.tool, toolState, trace, config) safeCapture(span) diff --git a/src/types.ts b/src/types.ts index 0d9d611..0a1bef0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -31,6 +31,10 @@ export interface TraceState { stepInputSnapshot: InputMessage[] /** Assistant text accumulated during the current step, reset on each step-start. */ stepAssistantText?: string + /** Names of tools called during the current step, in call order, reset on each step-start. */ + stepToolCalls: string[] + /** Tool call IDs already recorded for the current step. */ + stepToolCallIds: Set currentAssistantMsg?: AssistantInfo currentGenerationSpanId?: string agentName?: string