diff --git a/src/common/orpc/schemas/telemetry.ts b/src/common/orpc/schemas/telemetry.ts index 9997f6496..2865f2a48 100644 --- a/src/common/orpc/schemas/telemetry.ts +++ b/src/common/orpc/schemas/telemetry.ts @@ -78,6 +78,14 @@ const StreamCompletedPropertiesSchema = z.object({ output_tokens_b2: z.number(), }); +const CompactionCompletedPropertiesSchema = z.object({ + model: z.string(), + duration_b2: z.number(), + input_tokens_b2: z.number(), + output_tokens_b2: z.number(), + compaction_source: z.enum(["manual", "idle"]), +}); + const ProviderConfiguredPropertiesSchema = z.object({ provider: z.string(), keyType: z.string(), @@ -125,6 +133,10 @@ export const TelemetryEventSchema = z.discriminatedUnion("event", [ event: z.literal("stream_completed"), properties: StreamCompletedPropertiesSchema, }), + z.object({ + event: z.literal("compaction_completed"), + properties: CompactionCompletedPropertiesSchema, + }), z.object({ event: z.literal("provider_configured"), properties: ProviderConfiguredPropertiesSchema, diff --git a/src/common/telemetry/payload.ts b/src/common/telemetry/payload.ts index 1dfea2103..3fe6e2260 100644 --- a/src/common/telemetry/payload.ts +++ b/src/common/telemetry/payload.ts @@ -122,6 +122,22 @@ export interface StreamCompletedPayload { output_tokens_b2: number; } +/** + * Compaction completion event - tracks when history compaction finishes + */ +export interface CompactionCompletedPayload { + /** Model used for compaction */ + model: string; + /** Duration in seconds, rounded to nearest power of 2 */ + duration_b2: number; + /** Input tokens (pre-compaction history size), rounded to nearest power of 2 */ + input_tokens_b2: number; + /** Output tokens (post-compaction summary size), rounded to nearest power of 2 */ + output_tokens_b2: number; + /** Whether this compaction was user-triggered vs idle */ + compaction_source: "manual" | "idle"; +} + /** * Provider configuration event - tracks when users set up providers * Note: Only tracks that a key was set, never the actual value @@ -211,6 +227,7 @@ export type TelemetryEventPayload = | { event: "workspace_switched"; properties: WorkspaceSwitchedPayload } | { event: "message_sent"; properties: MessageSentPayload } | { event: "stream_completed"; properties: StreamCompletedPayload } + | { event: "compaction_completed"; properties: CompactionCompletedPayload } | { event: "provider_configured"; properties: ProviderConfiguredPayload } | { event: "command_used"; properties: CommandUsedPayload } | { event: "voice_transcription"; properties: VoiceTranscriptionPayload } diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 36a98e849..e106f79b5 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -30,6 +30,7 @@ import { createRuntime } from "@/node/runtime/runtimeFactory"; import { MessageQueue } from "./messageQueue"; import type { StreamEndEvent } from "@/common/types/stream"; import { CompactionHandler } from "./compactionHandler"; +import type { TelemetryService } from "./telemetryService"; import type { BackgroundProcessManager } from "./backgroundProcessManager"; import { computeDiff } from "@/node/utils/diff"; import { AttachmentService } from "./attachmentService"; @@ -89,6 +90,7 @@ interface AgentSessionOptions { partialService: PartialService; aiService: AIService; initStateManager: InitStateManager; + telemetryService?: TelemetryService; backgroundProcessManager: BackgroundProcessManager; /** Called when compaction completes (e.g., to clear idle compaction pending state) */ onCompactionComplete?: () => void; @@ -147,6 +149,7 @@ export class AgentSession { partialService, aiService, initStateManager, + telemetryService, backgroundProcessManager, onCompactionComplete, onPostCompactionStateChange, @@ -169,6 +172,7 @@ export class AgentSession { this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, historyService: this.historyService, + telemetryService, partialService: this.partialService, emitter: this.emitter, onCompactionComplete, diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 43adc2664..e7cdd2a4e 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -5,6 +5,8 @@ import type { PartialService } from "./partialService"; import type { EventEmitter } from "events"; import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import type { StreamEndEvent } from "@/common/types/stream"; +import type { TelemetryService } from "./telemetryService"; +import type { TelemetryEventPayload } from "@/common/telemetry/payload"; import { Ok, Err, type Result } from "@/common/types/result"; interface EmittedEvent { @@ -117,6 +119,8 @@ describe("CompactionHandler", () => { let mockHistoryService: ReturnType; let mockPartialService: ReturnType; let mockEmitter: EventEmitter; + let telemetryCapture: ReturnType; + let telemetryService: TelemetryService; let emittedEvents: EmittedEvent[]; const workspaceId = "test-workspace"; @@ -125,12 +129,18 @@ describe("CompactionHandler", () => { mockEmitter = emitter; emittedEvents = events; + telemetryCapture = mock((_payload: TelemetryEventPayload) => { + void _payload; + }); + telemetryService = { capture: telemetryCapture } as unknown as TelemetryService; + mockHistoryService = createMockHistoryService(); mockPartialService = createMockPartialService(); handler = new CompactionHandler({ workspaceId, historyService: mockHistoryService as unknown as HistoryService, + telemetryService, partialService: mockPartialService as unknown as PartialService, emitter: mockEmitter, }); @@ -160,6 +170,37 @@ describe("CompactionHandler", () => { expect(result).toBe(false); }); + it("should capture compaction_completed telemetry on successful compaction", async () => { + const compactionReq = createCompactionRequest(); + setupSuccessfulCompaction(mockHistoryService, [compactionReq]); + + const event = createStreamEndEvent("Summary", { + duration: 1500, + // Prefer contextUsage (context size) over total usage. + contextUsage: { inputTokens: 1000, outputTokens: 333, totalTokens: undefined }, + }); + + await handler.handleCompletion(event); + + expect(telemetryCapture.mock.calls).toHaveLength(1); + const payload = telemetryCapture.mock.calls[0][0] as TelemetryEventPayload; + expect(payload.event).toBe("compaction_completed"); + if (payload.event !== "compaction_completed") { + throw new Error("Expected compaction_completed payload"); + } + + expect(payload.properties).toEqual({ + model: "claude-3-5-sonnet-20241022", + // 1.5s -> 2 + duration_b2: 2, + // 1000 -> 1024 + input_tokens_b2: 1024, + // 333 -> 512 + output_tokens_b2: 512, + compaction_source: "manual", + }); + }); + it("should return true when successful", async () => { const compactionReq = createCompactionRequest(); mockHistoryService.mockGetHistory(Ok([compactionReq])); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 711efada2..18e77ebc1 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -9,6 +9,8 @@ import { Ok, Err } from "@/common/types/result"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import { createMuxMessage, type MuxMessage } from "@/common/types/message"; +import type { TelemetryService } from "@/node/services/telemetryService"; +import { roundToBase2 } from "@/common/telemetry/utils"; import { log } from "@/node/services/log"; import { extractEditedFileDiffs, @@ -20,6 +22,7 @@ interface CompactionHandlerOptions { workspaceId: string; historyService: HistoryService; partialService: PartialService; + telemetryService?: TelemetryService; emitter: EventEmitter; /** Called when compaction completes successfully (e.g., to clear idle compaction pending state) */ onCompactionComplete?: () => void; @@ -37,6 +40,7 @@ export class CompactionHandler { private readonly workspaceId: string; private readonly historyService: HistoryService; private readonly partialService: PartialService; + private readonly telemetryService?: TelemetryService; private readonly emitter: EventEmitter; private readonly processedCompactionRequestIds: Set = new Set(); private readonly onCompactionComplete?: () => void; @@ -50,6 +54,7 @@ export class CompactionHandler { this.workspaceId = options.workspaceId; this.historyService = options.historyService; this.partialService = options.partialService; + this.telemetryService = options.telemetryService; this.emitter = options.emitter; this.onCompactionComplete = options.onCompactionComplete; } @@ -130,6 +135,24 @@ export class CompactionHandler { return false; } + const durationSecs = + typeof event.metadata.duration === "number" ? event.metadata.duration / 1000 : 0; + const inputTokens = + event.metadata.contextUsage?.inputTokens ?? event.metadata.usage?.inputTokens ?? 0; + const outputTokens = + event.metadata.contextUsage?.outputTokens ?? event.metadata.usage?.outputTokens ?? 0; + + this.telemetryService?.capture({ + event: "compaction_completed", + properties: { + model: event.metadata.model, + duration_b2: roundToBase2(durationSecs), + input_tokens_b2: roundToBase2(inputTokens ?? 0), + output_tokens_b2: roundToBase2(outputTokens ?? 0), + compaction_source: isIdleCompaction ? "idle" : "manual", + }, + }); + // Notify that compaction completed (clears idle compaction pending state) this.onCompactionComplete?.(); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 8701e8007..e7e476eb9 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -116,6 +116,7 @@ export class ServiceContainer { this.menuEventService = new MenuEventService(); this.voiceService = new VoiceService(config); this.telemetryService = new TelemetryService(config.rootDir); + this.workspaceService.setTelemetryService(this.telemetryService); this.experimentsService = new ExperimentsService({ telemetryService: this.telemetryService, muxHome: config.rootDir, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c5a34892c..6008c2be5 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13,6 +13,7 @@ import type { PartialService } from "@/node/services/partialService"; import type { AIService } from "@/node/services/aiService"; import type { InitStateManager } from "@/node/services/initStateManager"; import type { ExtensionMetadataService } from "@/node/services/ExtensionMetadataService"; +import type { TelemetryService } from "@/node/services/telemetryService"; import type { ExperimentsService } from "@/node/services/experimentsService"; import { EXPERIMENT_IDS, EXPERIMENTS } from "@/common/constants/experiments"; import type { MCPServerManager } from "@/node/services/mcpServerManager"; @@ -120,6 +121,7 @@ export class WorkspaceService extends EventEmitter { this.setupMetadataListeners(); } + private telemetryService?: TelemetryService; private experimentsService?: ExperimentsService; private mcpServerManager?: MCPServerManager; // Optional terminal service for cleanup on workspace removal @@ -133,6 +135,10 @@ export class WorkspaceService extends EventEmitter { this.mcpServerManager = manager; } + setTelemetryService(telemetryService: TelemetryService): void { + this.telemetryService = telemetryService; + } + setExperimentsService(experimentsService: ExperimentsService): void { this.experimentsService = experimentsService; } @@ -301,6 +307,7 @@ export class WorkspaceService extends EventEmitter { historyService: this.historyService, partialService: this.partialService, aiService: this.aiService, + telemetryService: this.telemetryService, initStateManager: this.initStateManager, backgroundProcessManager: this.backgroundProcessManager, onCompactionComplete: () => {