Skip to content

Commit 7e154b7

Browse files
carderneTrigger.dev RepoOps
authored andcommitted
fix(billing): refresh usage credentials through worker heartbeats
Mono-RevId: 509005f66c0113ba6d6f66d6fcf341fd474d3b19
1 parent b1826ab commit 7e154b7

25 files changed

Lines changed: 1254 additions & 53 deletions

‎apps/webapp/app/env.server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1289,6 +1289,8 @@ const EnvironmentSchema = z
12891289
/* Usage settings */
12901290
USAGE_EVENT_URL: z.string().optional(),
12911291
PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(),
1292+
/** "1" lets runs resend usage reports that got no definite answer. Turn on only with billing dedup enabled. */
1293+
USAGE_RETRY_AMBIGUOUS_REPORTS: z.enum(["0", "1"]).default("0"),
12921294

12931295
CENTS_PER_RUN: z.coerce.number().default(0),
12941296

‎apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export const action = createActionWorkerApiRoute(
3030
workerCell: body.workerCell,
3131
workerLane: body.workerLane,
3232
supportsRunTokenRefresh: body.supportsRunTokenRefresh,
33+
supportsUsageJwtRefresh: body.supportsUsageJwtRefresh,
3334
supportsSecretProxy: body.supportsSecretProxy,
3435
runnerId,
3536
environmentId,

‎apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.heartbeat.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,16 @@ export const action = createActionWorkerApiRoute(
2828
}): Promise<TypedResponse<WorkerApiRunHeartbeatResponseBody>> => {
2929
const { runFriendlyId, snapshotFriendlyId } = params;
3030

31-
const { runToken } = await authenticatedWorker.heartbeatRun({
31+
const { runToken, usageJwt, usageJwtError } = await authenticatedWorker.heartbeatRun({
3232
runFriendlyId,
3333
snapshotFriendlyId,
3434
runnerId,
3535
environmentId,
3636
sandboxRunId,
3737
runTokenReport: body.runToken,
38+
usageJwtReport: body.usageJwt,
3839
});
3940

40-
return json({ ok: true, runToken });
41+
return json({ ok: true, runToken, usageJwt, usageJwtError });
4142
}
4243
);

‎apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2143,6 +2143,10 @@ async function resolveBuiltInProdVariables(
21432143
value: env.USAGE_EVENT_URL,
21442144
},
21452145
]);
2146+
2147+
if (env.USAGE_RETRY_AMBIGUOUS_REPORTS === "1") {
2148+
result = result.concat([{ key: "USAGE_RETRY_AMBIGUOUS_REPORTS", value: "1" }]);
2149+
}
21462150
}
21472151

21482152
if (env.PROD_TASK_HEARTBEAT_INTERVAL_MS) {
Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
import type { RunStore } from "@internal/run-store";
2+
import { getMeter } from "@internal/tracing";
3+
import {
4+
fromFriendlyId,
5+
type TaskRunContext,
6+
type TaskRunExecutionStatus,
7+
} from "@trigger.dev/core/v3";
8+
import {
9+
USAGE_JWT_REFRESH_BEFORE_EXPIRY_SECONDS,
10+
type WorkerApiRunHeartbeatRequestBody,
11+
} from "@trigger.dev/core/v3/workers";
12+
import { singleton } from "~/utils/singleton";
13+
import { meteringClaims } from "~/v3/utils/meteringClaims.server";
14+
15+
type EnvironmentIdentity = { id: string; organizationId: string; projectId: string };
16+
17+
const LIVE_ATTEMPT_STATUSES: ReadonlySet<TaskRunExecutionStatus> = new Set<TaskRunExecutionStatus>([
18+
"PENDING_EXECUTING",
19+
"EXECUTING",
20+
"EXECUTING_WITH_WAITPOINTS",
21+
"QUEUED_EXECUTING",
22+
]);
23+
24+
const REFUSAL_REASONS = ["missing_identity", "not_owner", "identity_changed"] as const;
25+
type RefusalReason = (typeof REFUSAL_REASONS)[number];
26+
27+
class UsageJwtRefusal extends Error {
28+
constructor(readonly reason: RefusalReason) {
29+
super(`Billing JWT refresh refused: ${reason}`);
30+
}
31+
}
32+
33+
// Names and label values are relied on by internal dashboards and monitors. Labels stay bounded.
34+
const telemetry = singleton("usageJwtRefreshTelemetry", () => {
35+
const issued = getMeter("usage-jwt-refresh").createCounter("usage_jwt.refresh.issued", {
36+
description: "Due billing JWT refresh requests handled by the webapp. Not delivery success",
37+
});
38+
// Export the failure series from zero, so a rate query sees a process's first failure.
39+
for (const reason of REFUSAL_REASONS) issued.add(0, { result: "refused", reason });
40+
issued.add(0, { result: "error", reason: "error" });
41+
const initial = getMeter("usage-jwt-refresh").createCounter("usage_jwt.initial", {
42+
description: "Billing credential kind handed out at attempt start",
43+
});
44+
initial.add(0, { kind: "legacy_unverified" });
45+
return { issued, initial };
46+
});
47+
48+
export type InitialUsageAuth =
49+
| "control_plane"
50+
| "legacy_runner"
51+
| "legacy_root_key"
52+
| "legacy_unverified";
53+
54+
/**
55+
* Which billing credential an attempt starts with. A run holding the root key keeps API-key
56+
* recovery; only a run without it needs the control plane to renew its credential.
57+
*/
58+
export function initialUsageAuth(opts: {
59+
supportsUsageJwtRefresh?: boolean;
60+
verifiedEnvironmentId?: string;
61+
environmentId: string;
62+
hasRootKey: boolean;
63+
}): InitialUsageAuth {
64+
if (!opts.supportsUsageJwtRefresh) return "legacy_runner";
65+
if (opts.hasRootKey) return "legacy_root_key";
66+
if (opts.verifiedEnvironmentId !== opts.environmentId) return "legacy_unverified";
67+
return "control_plane";
68+
}
69+
70+
export function countInitialUsageAuth(kind: InitialUsageAuth) {
71+
telemetry.initial.add(1, { kind });
72+
}
73+
74+
type RefreshOptions = {
75+
report: WorkerApiRunHeartbeatRequestBody["usageJwt"];
76+
runId: string;
77+
snapshotId: string;
78+
workerId: string;
79+
runnerId?: string;
80+
environmentId?: string;
81+
projectId?: string;
82+
store: Pick<RunStore, "findLatestExecutionSnapshot" | "findRun">;
83+
resolveContext: (runId: string) => Promise<TaskRunContext>;
84+
sign: (environment: EnvironmentIdentity, claims: Record<string, string>) => Promise<string>;
85+
now?: number;
86+
};
87+
88+
/** Mints a replacement only when the reported credential is due. Throws when refused or failed. */
89+
export async function refreshUsageJwtOnHeartbeat(
90+
opts: RefreshOptions
91+
): Promise<string | undefined> {
92+
const { report } = opts;
93+
if (!report) return undefined;
94+
if (
95+
report.expiresAt - (opts.now ?? Date.now()) / 1_000 >
96+
USAGE_JWT_REFRESH_BEFORE_EXPIRY_SECONDS
97+
) {
98+
return undefined;
99+
}
100+
101+
try {
102+
const token = await issue(opts, report);
103+
telemetry.issued.add(1, { result: "success", reason: "none" });
104+
return token;
105+
} catch (error) {
106+
const refused = error instanceof UsageJwtRefusal;
107+
telemetry.issued.add(1, {
108+
result: refused ? "refused" : "error",
109+
reason: refused ? error.reason : "error",
110+
});
111+
throw error;
112+
}
113+
}
114+
115+
async function issue(
116+
opts: RefreshOptions,
117+
report: NonNullable<RefreshOptions["report"]>
118+
): Promise<string> {
119+
if (!opts.environmentId || !opts.runnerId) throw new UsageJwtRefusal("missing_identity");
120+
121+
const assertOwnership = async () => {
122+
const snapshot = await opts.store.findLatestExecutionSnapshot(
123+
opts.runId,
124+
undefined,
125+
opts.environmentId
126+
);
127+
if (
128+
!snapshot ||
129+
!snapshot.isValid ||
130+
snapshot.id !== opts.snapshotId ||
131+
snapshot.runId !== opts.runId ||
132+
snapshot.environmentId !== opts.environmentId ||
133+
snapshot.workerId !== opts.workerId ||
134+
snapshot.runnerId !== opts.runnerId ||
135+
snapshot.attemptNumber !== report.attemptNumber ||
136+
!LIVE_ATTEMPT_STATUSES.has(snapshot.executionStatus) ||
137+
(opts.projectId !== undefined && snapshot.projectId !== opts.projectId)
138+
) {
139+
throw new UsageJwtRefusal("not_owner");
140+
}
141+
return snapshot;
142+
};
143+
144+
const snapshot = await assertOwnership();
145+
const [context, run] = await Promise.all([
146+
opts.resolveContext(opts.runId),
147+
opts.store.findRun(
148+
{ id: opts.runId },
149+
{
150+
select: {
151+
runtimeEnvironmentId: true,
152+
attemptNumber: true,
153+
region: true,
154+
workerQueue: true,
155+
},
156+
}
157+
),
158+
]);
159+
if (
160+
!run ||
161+
run.runtimeEnvironmentId !== snapshot.environmentId ||
162+
run.attemptNumber !== report.attemptNumber ||
163+
fromFriendlyId(context.run.id) !== opts.runId ||
164+
context.environment.id !== snapshot.environmentId ||
165+
context.organization.id !== snapshot.organizationId ||
166+
context.project.id !== snapshot.projectId ||
167+
context.attempt.number !== report.attemptNumber
168+
) {
169+
throw new UsageJwtRefusal("identity_changed");
170+
}
171+
// Context resolution can cross stores. Do not sign if the attempt moved during those reads.
172+
await assertOwnership();
173+
174+
return opts.sign(
175+
{
176+
id: context.environment.id,
177+
organizationId: context.organization.id,
178+
projectId: context.project.id,
179+
},
180+
{
181+
...meteringClaims({
182+
environmentType: context.environment.type,
183+
region: run.region,
184+
workerQueue: run.workerQueue,
185+
}),
186+
run_id: opts.runId,
187+
machine_preset: context.machine.name,
188+
usage_auth: "control-plane-v1",
189+
usage_attempt: String(report.attemptNumber),
190+
}
191+
);
192+
}

‎apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts‎

Lines changed: 83 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { fromFriendlyId } from "@trigger.dev/core/v3/isomorphic";
1919
import {
2020
WORKER_HEADERS,
2121
type RunTokenHeartbeatReport,
22+
type WorkerApiRunHeartbeatRequestBody,
2223
type WeightedWorkerQueueSubscription,
2324
type WorkerQueueClass,
2425
} from "@trigger.dev/core/v3/workers";
@@ -66,6 +67,11 @@ import type { EnvironmentVariable } from "~/v3/environmentVariables/repository";
6667
import { machinePresetFromName } from "~/v3/machinePresets.server";
6768
import { meteringClaims } from "~/v3/utils/meteringClaims.server";
6869
import { issueContinueRunToken, refreshRunTokenOnHeartbeat } from "./runTokenRefresh.server";
70+
import {
71+
countInitialUsageAuth,
72+
initialUsageAuth,
73+
refreshUsageJwtOnHeartbeat,
74+
} from "./usageJwtRefresh.server";
6975
import type { WithRunEngineOptions } from "../baseService.server";
7076
import { WithRunEngine } from "../baseService.server";
7177

@@ -602,14 +608,16 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
602608
environmentId,
603609
sandboxRunId,
604610
runTokenReport,
611+
usageJwtReport,
605612
}: {
606613
runFriendlyId: string;
607614
snapshotFriendlyId: string;
608615
runnerId?: string;
609616
environmentId?: string;
610617
sandboxRunId?: string;
611618
runTokenReport?: RunTokenHeartbeatReport;
612-
}): Promise<{ runToken?: string }> {
619+
usageJwtReport?: WorkerApiRunHeartbeatRequestBody["usageJwt"];
620+
}): Promise<{ runToken?: string; usageJwt?: string; usageJwtError?: "refresh_failed" }> {
613621
const heartbeat = await this._engine.heartbeatRun({
614622
runId: fromFriendlyId(runFriendlyId),
615623
snapshotId: fromFriendlyId(snapshotFriendlyId),
@@ -638,7 +646,33 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
638646
replica: this._replica,
639647
});
640648

641-
return { runToken };
649+
let usageJwt: string | undefined;
650+
let usageJwtError: "refresh_failed" | undefined;
651+
try {
652+
usageJwt = await refreshUsageJwtOnHeartbeat({
653+
report: usageJwtReport,
654+
runId: fromFriendlyId(runFriendlyId),
655+
snapshotId: fromFriendlyId(snapshotFriendlyId),
656+
workerId: this.workerInstanceId,
657+
runnerId,
658+
environmentId,
659+
projectId: this.type === "UNMANAGED" ? (this.projectId ?? undefined) : undefined,
660+
store: this._engine.runStore,
661+
resolveContext: (runId) => this._engine.resolveTaskRunContext(runId),
662+
sign: generateJWTTokenForEnvironment,
663+
});
664+
} catch (error) {
665+
usageJwtError = "refresh_failed";
666+
logger.warn("Billing JWT refresh rejected or failed", {
667+
runId: runFriendlyId,
668+
snapshotId: snapshotFriendlyId,
669+
workerId: this.workerInstanceId,
670+
runnerId,
671+
error: error instanceof Error ? error.message : String(error),
672+
});
673+
}
674+
675+
return { runToken, usageJwt, usageJwtError };
642676
}
643677

644678
async startRunAttempt({
@@ -651,6 +685,7 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
651685
environmentId,
652686
sandboxRunId,
653687
supportsRunTokenRefresh,
688+
supportsUsageJwtRefresh,
654689
supportsSecretProxy,
655690
}: {
656691
runFriendlyId: string;
@@ -662,6 +697,7 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
662697
environmentId?: string;
663698
sandboxRunId?: string;
664699
supportsRunTokenRefresh?: boolean;
700+
supportsUsageJwtRefresh?: boolean;
665701
supportsSecretProxy?: boolean;
666702
}): Promise<
667703
StartRunAttemptResult & {
@@ -758,16 +794,32 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
758794
variables = [...variables, ...delivery.placeholders];
759795
}
760796

797+
const usageAuth = this.initialUsageAuthFor({
798+
runId: engineResult.run.id,
799+
supportsUsageJwtRefresh,
800+
verifiedEnvironmentId: environmentId,
801+
environmentId: environment.id,
802+
hasRootKey: variables.some((variable) => variable.key === "TRIGGER_SECRET_KEY"),
803+
});
804+
761805
const envVars = await this.getEnvVars(
762806
variables,
763807
environment,
764808
engineResult.run.id,
765809
engineResult.execution.machine ?? defaultMachinePreset,
766-
meteringClaims({
767-
environmentType: environment.type,
768-
region: engineResult.run.region,
769-
workerQueue: engineResult.run.workerQueue,
770-
}),
810+
{
811+
...meteringClaims({
812+
environmentType: environment.type,
813+
region: engineResult.run.region,
814+
workerQueue: engineResult.run.workerQueue,
815+
}),
816+
...(usageAuth === "control_plane"
817+
? {
818+
usage_auth: "control-plane-v1",
819+
usage_attempt: String(engineResult.execution.attempt.number),
820+
}
821+
: {}),
822+
},
771823
engineResult.run.taskEventStore ?? undefined,
772824
runToken
773825
);
@@ -779,6 +831,30 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
779831
};
780832
}
781833

834+
private initialUsageAuthFor(opts: {
835+
runId: string;
836+
supportsUsageJwtRefresh?: boolean;
837+
verifiedEnvironmentId?: string;
838+
environmentId: string;
839+
hasRootKey: boolean;
840+
}) {
841+
const kind = initialUsageAuth(opts);
842+
countInitialUsageAuth(kind);
843+
844+
// A run without the root key whose environment the supervisor did not verify: its credential
845+
// cannot be renewed by either path. This is the alert signal.
846+
if (kind === "legacy_unverified") {
847+
logger.warn("Billing JWT issued without any renewal path: unverified environment", {
848+
runId: opts.runId,
849+
environmentId: opts.environmentId,
850+
verifiedEnvironmentId: opts.verifiedEnvironmentId,
851+
workerId: this.workerInstanceId,
852+
});
853+
}
854+
855+
return kind;
856+
}
857+
782858
// The engine validates the header's scope before any resolved variables are returned.
783859
private async findAttemptEnvironment(runId: string, environmentId?: string) {
784860
const where = { id: runId };

0 commit comments

Comments
 (0)