|
| 1 | +import { expect, test } from '@playwright/test'; |
| 2 | +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; |
| 3 | +import { newAgentId, prompt, submitPrompt, waitForOperation } from './utils'; |
| 4 | + |
| 5 | +const APP = 'cloudflare-pi-durable'; |
| 6 | + |
| 7 | +test('traces a PiHarness prompt as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => { |
| 8 | + // A run is its own trace, and its `invoke_agent` segment ends last. Only this test calls get_weather. |
| 9 | + const spansPromise = collectStreamedSpans( |
| 10 | + APP, |
| 11 | + spansOfTrace => |
| 12 | + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === 'gen_ai.invoke_agent') && |
| 13 | + spansOfTrace.some(span => span.attributes['gen_ai.tool.name']?.value === 'get_weather'), |
| 14 | + ); |
| 15 | + |
| 16 | + const answer = await prompt( |
| 17 | + baseURL!, |
| 18 | + newAgentId('weather'), |
| 19 | + 'What is the weather in Vienna? Use the get_weather tool.', |
| 20 | + ); |
| 21 | + expect(answer.status, answer.reason).toBe('done'); |
| 22 | + |
| 23 | + const spans = await spansPromise; |
| 24 | + const agent = spans.find(span => span.is_segment)!; |
| 25 | + const chats = spans.filter(span => getSpanOp(span) === 'gen_ai.chat'); |
| 26 | + // The manual span of the tool is the one way to pick the call that ran, should the model call twice. |
| 27 | + const manualSpan = spans.find(span => span.name === 'resolve-weather')!; |
| 28 | + const tool = spans.find(span => span.span_id === manualSpan.parent_span_id)!; |
| 29 | + const providerCalls = spans.filter(span => getSpanOp(span) === 'http.client'); |
| 30 | + |
| 31 | + expect(agent.attributes['sentry.origin']?.value).toBe('auto.ai.pi_durable'); |
| 32 | + // The PiHarness root session is pi conversation 1, in a Harness of its own per Durable Object. |
| 33 | + expect(agent.attributes['gen_ai.conversation.id']?.value).toMatch(/^[0-9a-f]{32}:1$/); |
| 34 | + // The request that submitted the prompt is a trace of its own: the scheduler runs the work. |
| 35 | + expect(spans.some(span => getSpanOp(span) === 'http.server')).toBe(false); |
| 36 | + // pi-durable keeps its state in the `pi_` tables of `PiHarness`, whose statements start no span. |
| 37 | + expect(spans.filter(span => getSpanOp(span) === 'db.query' && /\bpi_/.test(String(span.name)))).toEqual([]); |
| 38 | + // pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so |
| 39 | + // each request is reported once. |
| 40 | + expect(spans.filter(span => String(span.attributes['sentry.origin']?.value).startsWith('auto.ai.'))).toEqual( |
| 41 | + spans.filter(span => span.attributes['sentry.origin']?.value === 'auto.ai.pi_durable'), |
| 42 | + ); |
| 43 | + |
| 44 | + // One tool-calling response, then the answer. |
| 45 | + expect(chats.length).toBeGreaterThanOrEqual(2); |
| 46 | + for (const chat of chats) { |
| 47 | + expect(chat.parent_span_id).toBe(agent.span_id); |
| 48 | + expect(chat.attributes['gen_ai.provider.name']?.value).toBe('openrouter'); |
| 49 | + expect(chat.attributes['gen_ai.request.model']?.value).toBe('anthropic/claude-haiku-4.5'); |
| 50 | + expect(chat.attributes['gen_ai.conversation.id']?.value).toBe(agent.attributes['gen_ai.conversation.id']?.value); |
| 51 | + // pi-durable retries a provider error inside the run; such a request has no usage and no HTTP span |
| 52 | + // of its own to assert on. |
| 53 | + if (chat.status === 'ok') { |
| 54 | + expect(typeof chat.attributes['gen_ai.usage.input_tokens']?.value).toBe('number'); |
| 55 | + expect(typeof chat.attributes['gen_ai.usage.output_tokens']?.value).toBe('number'); |
| 56 | + expect(providerCalls.some(providerCall => providerCall.parent_span_id === chat.span_id)).toBe(true); |
| 57 | + } |
| 58 | + } |
| 59 | + |
| 60 | + expect(tool.attributes['gen_ai.tool.name']?.value).toBe('get_weather'); |
| 61 | + expect(tool.parent_span_id).toBe(agent.span_id); |
| 62 | + expect(tool.status).toBe('ok'); |
| 63 | + expect(tool.attributes['gen_ai.tool.call.arguments']?.value).toContain('Vienna'); |
| 64 | + expect(tool.attributes['gen_ai.tool.call.result']?.value).toContain('21 degrees and sunny in'); |
| 65 | + |
| 66 | + // The provider's HTTP calls nest inside the `chat` span that sent them. |
| 67 | + expect(providerCalls.length).toBeGreaterThan(0); |
| 68 | + for (const providerCall of providerCalls) { |
| 69 | + expect(chats.map(chat => chat.span_id)).toContain(providerCall.parent_span_id); |
| 70 | + expect(providerCall.attributes['server.address']?.value).toBe('openrouter.ai'); |
| 71 | + } |
| 72 | +}); |
| 73 | + |
| 74 | +test('reports a throwing tool as an error on its execute_tool span', async ({ baseURL }) => { |
| 75 | + const errorPromise = waitForError( |
| 76 | + APP, |
| 77 | + event => event.exception?.values?.[0]?.value === 'Intentional pi-durable tool failure', |
| 78 | + ); |
| 79 | + const spansPromise = collectStreamedSpans( |
| 80 | + APP, |
| 81 | + spansOfTrace => |
| 82 | + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === 'gen_ai.invoke_agent') && |
| 83 | + spansOfTrace.some(span => span.attributes['gen_ai.tool.name']?.value === 'fail_now'), |
| 84 | + ); |
| 85 | + |
| 86 | + const answer = await prompt(baseURL!, newAgentId('fail'), 'Call the fail_now tool, then tell me what happened.'); |
| 87 | + expect(answer.status, answer.reason).toBe('done'); |
| 88 | + |
| 89 | + const [error, spans] = await Promise.all([errorPromise, spansPromise]); |
| 90 | + // The error is reported on the span of the call that threw. |
| 91 | + const tool = spans.find(span => span.span_id === error.contexts?.trace?.span_id)!; |
| 92 | + |
| 93 | + expect(tool.attributes['gen_ai.tool.name']?.value).toBe('fail_now'); |
| 94 | + expect(tool.status).toBe('error'); |
| 95 | + expect(tool.trace_id).toBe(error.contexts?.trace?.trace_id); |
| 96 | + expect(error.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: true }); |
| 97 | +}); |
| 98 | + |
| 99 | +test('resumes a run in a new trace after the Durable Object resets during a tool call', async ({ baseURL }) => { |
| 100 | + const agentId = newAgentId('reset'); |
| 101 | + const operationId = crypto.randomUUID(); |
| 102 | + // The tool span of the reset object never ends, so a finished `crash_once` span can only come |
| 103 | + // from the rerun after the restart. |
| 104 | + const spansPromise = collectStreamedSpans( |
| 105 | + APP, |
| 106 | + spansOfTrace => |
| 107 | + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === 'gen_ai.invoke_agent') && |
| 108 | + spansOfTrace.some(span => span.attributes['gen_ai.tool.name']?.value === 'crash_once' && span.status === 'ok'), |
| 109 | + ); |
| 110 | + |
| 111 | + // The reset fails the request that submitted the prompt, but pi has stored the input already. |
| 112 | + await submitPrompt( |
| 113 | + baseURL!, |
| 114 | + agentId, |
| 115 | + 'Run the crash_once tool for job "nightly", then reply with the word DONE.', |
| 116 | + operationId, |
| 117 | + ); |
| 118 | + const answer = await waitForOperation(baseURL!, agentId, operationId); |
| 119 | + expect(answer.status, answer.reason).toBe('done'); |
| 120 | + |
| 121 | + const spans = await spansPromise; |
| 122 | + const agent = spans.find(span => span.is_segment)!; |
| 123 | + const tool = spans.find(span => span.attributes['gen_ai.tool.name']?.value === 'crash_once')!; |
| 124 | + const chats = spans.filter(span => getSpanOp(span) === 'gen_ai.chat'); |
| 125 | + const finalAnswer = chats.find(span => span.attributes['gen_ai.response.finish_reasons']?.value === '["stop"]'); |
| 126 | + |
| 127 | + expect(getSpanOp(agent)).toBe('gen_ai.invoke_agent'); |
| 128 | + expect(tool.parent_span_id).toBe(agent.span_id); |
| 129 | + expect(finalAnswer?.parent_span_id).toBe(agent.span_id); |
| 130 | + // The request that called the tool ran before the reset, so this trace starts with the rerun of the |
| 131 | + // tool. A run that never reset would start with that request. |
| 132 | + expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true); |
| 133 | +}); |
0 commit comments