feat: consecutive-window alerts

.changeset/rare-spoons-change.md

@@ -0,0 +1,9 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+Adding consecutive-window configuration to alerts, so that you can specify a condition like "only fire this alert after some condition is met for N consecutive windows." This helps prevent flaky alerts (and pages), and cut down on alert noise in many cases.
+
+Also adds a `PENDING` alert state for alarms that _will_ fire if current trends continue.

packages/api/src/controllers/alertHistory.ts

@@ -18,14 +18,24 @@ type GroupedAlertHistory = {
   lastValues: IAlertHistory['lastValues'][];
 };
 
+function groupStateToOverallState(states: string[]): AlertState {
+  if (states.includes(AlertState.ALERT)) {
+    return AlertState.ALERT;
+  }
+
+  if (states.includes(AlertState.PENDING)) {
+    return AlertState.PENDING;
+  }
+
+  return AlertState.OK;
+}
+
 function mapGroupedHistories(
   groupedHistories: GroupedAlertHistory[],
 ): Omit<IAlertHistory, 'alert'>[] {
   return groupedHistories.map(group => ({
     createdAt: group._id,
-    state: group.states.includes(AlertState.ALERT)
-      ? AlertState.ALERT
-      : AlertState.OK,
+    state: groupStateToOverallState(group.states),
     counts: group.counts,
     lastValues: group.lastValues
       .flat()

packages/api/src/controllers/alerts.ts

@@ -155,6 +155,9 @@ const makeAlert = (alert: AlertInput, userId?: ObjectId): Partial<IAlert> => {
     // Chart alerts
     dashboard: alert.dashboardId as unknown as ObjectId,
     tileId: alert.tileId,
+
+    // Multi-window alerting
+    numConsecutiveWindows: alert.numConsecutiveWindows ?? null,
   };
 };
 

packages/api/src/models/alert.ts

@@ -14,6 +14,7 @@ export enum AlertState {
   DISABLED = 'DISABLED',
   INSUFFICIENT_DATA = 'INSUFFICIENT_DATA',
   OK = 'OK',
+  PENDING = 'PENDING',
 }
 
 export interface IAlertError {
@@ -84,6 +85,9 @@ export interface IAlert {
     until: Date;
   };
 
+  // Multi-window alerting: fire only after N violations in M consecutive windows
+  numConsecutiveWindows?: number | null;
+
   // Errors recorded during the most recent execution
   executionErrors?: IAlertError[];
   createdAt: Date;
@@ -190,6 +194,11 @@ const AlertSchema = new Schema<IAlert>(
       type: String,
       required: false,
     },
+    numConsecutiveWindows: {
+      type: Number,
+      required: false,
+      min: 1,
+    },
     silenced: {
       required: false,
       type: {

packages/api/src/models/alertHistory.ts

@@ -12,6 +12,7 @@ export interface IAlertHistory {
   state: AlertState;
   lastValues: { startTime: Date; count: number }[];
   group?: string; // For group-by alerts, stores the group identifier
+  fired?: boolean;
 }
 
 const AlertHistorySchema = new Schema<IAlertHistory>({
@@ -45,6 +46,10 @@ const AlertHistorySchema = new Schema<IAlertHistory>({
     type: String,
     required: false,
   },
+  fired: {
+    type: Boolean,
+    required: false,
+  },
 });
 
 AlertHistorySchema.index(

packages/api/src/routers/api/alerts.ts

@@ -84,6 +84,7 @@ const formatAlertResponse = (
       'createdAt',
       'updatedAt',
       'executionErrors',
+      'numConsecutiveWindows',
     ]),
   };
 };