🤖 feat: simplify dev telemetry env var and track experiment overrides

- Makefile: MUX_ENABLE_TELEMETRY_IN_DEV=1 now sufficient (no need to also set MUX_DISABLE_TELEMETRY=0)
- ExperimentsSection: hide non-overridable experiments, remove PostHog info line
- Add experiment_overridden telemetry event with experimentId, assignedVariant, userChoice
- Update oRPC schema, payload types, tracking functions, and useTelemetry hook

Signed-off-by: Thomas Kosiewski <[email protected]>

---
_Generated with `mux` • Model: `anthropic:claude-opus-4-5` • Thinking: `high`_

Change-Id: I3582117b82c1025bcfd94d1361bb11c46cb8ff9e

Author: ThomasK33

Makefile

@@ -131,13 +131,13 @@ dev: node_modules/.installed build-main ## Start development server (Vite + node
 	@echo "Starting dev mode (3 watchers: nodemon for main process, esbuild for api, vite for renderer)..."
 	# On Windows, use npm run because bunx doesn't correctly pass arguments to concurrently
 	# https://github.com/oven-sh/bun/issues/18275
-	@MUX_DISABLE_TELEMETRY=$(or $(MUX_DISABLE_TELEMETRY),1) NODE_OPTIONS="--max-old-space-size=4096" npm x concurrently -k --raw \
+	@MUX_DISABLE_TELEMETRY=$(if $(MUX_ENABLE_TELEMETRY_IN_DEV),,$(or $(MUX_DISABLE_TELEMETRY),1)) NODE_OPTIONS="--max-old-space-size=4096" npm x concurrently -k --raw \
 		"bun x nodemon --watch src --watch tsconfig.main.json --watch tsconfig.json --ext ts,tsx,json --ignore dist --ignore node_modules --exec node scripts/build-main-watch.js" \
 		"npx esbuild src/cli/api.ts --bundle --format=esm --platform=node --target=node20 --outfile=dist/cli/api.mjs --external:zod --external:commander --external:@trpc/server --watch" \
 		"vite"
 else
 dev: node_modules/.installed build-main build-preload ## Start development server (Vite + tsgo watcher for 10x faster type checking)
-	@MUX_DISABLE_TELEMETRY=$(or $(MUX_DISABLE_TELEMETRY),1) bun x concurrently -k \
+	@MUX_DISABLE_TELEMETRY=$(if $(MUX_ENABLE_TELEMETRY_IN_DEV),,$(or $(MUX_DISABLE_TELEMETRY),1)) bun x concurrently -k \
 		"bun x concurrently \"$(TSGO) -w -p tsconfig.main.json\" \"bun x tsc-alias -w -p tsconfig.main.json\"" \
 		"bun x esbuild src/cli/api.ts --bundle --format=esm --platform=node --target=node20 --outfile=dist/cli/api.mjs --external:zod --external:commander --external:@trpc/server --watch" \
 		"vite"
@@ -151,7 +151,7 @@ dev-server: node_modules/.installed build-main ## Start server mode with hot rel
 	@echo ""
 	@echo "For remote access: make dev-server VITE_HOST=0.0.0.0 BACKEND_HOST=0.0.0.0"
 	@# On Windows, use npm run because bunx doesn't correctly pass arguments
-	@MUX_DISABLE_TELEMETRY=$(or $(MUX_DISABLE_TELEMETRY),1) npmx concurrently -k \
+	@MUX_DISABLE_TELEMETRY=$(if $(MUX_ENABLE_TELEMETRY_IN_DEV),,$(or $(MUX_DISABLE_TELEMETRY),1)) npmx concurrently -k \
 		"npmx nodemon --watch src --watch tsconfig.main.json --watch tsconfig.json --ext ts,tsx,json --ignore dist --ignore node_modules --exec node scripts/build-main-watch.js" \
 		"npx esbuild src/cli/api.ts --bundle --format=esm --platform=node --target=node20 --outfile=dist/cli/api.mjs --external:zod --external:commander --external:@trpc/server --watch" \
 		"npmx nodemon --watch dist/cli/index.js --watch dist/cli/server.js --delay 500ms --exec \"node dist/cli/index.js server --host $(or $(BACKEND_HOST),localhost) --port $(or $(BACKEND_PORT),3000)\"" \
@@ -163,7 +163,7 @@ dev-server: node_modules/.installed build-main ## Start server mode with hot rel
 	@echo "  Frontend (with HMR):     http://$(or $(VITE_HOST),localhost):$(or $(VITE_PORT),5173)"
 	@echo ""
 	@echo "For remote access: make dev-server VITE_HOST=0.0.0.0 BACKEND_HOST=0.0.0.0"
-	@MUX_DISABLE_TELEMETRY=$(or $(MUX_DISABLE_TELEMETRY),1) bun x concurrently -k \
+	@MUX_DISABLE_TELEMETRY=$(if $(MUX_ENABLE_TELEMETRY_IN_DEV),,$(or $(MUX_DISABLE_TELEMETRY),1)) bun x concurrently -k \
 		"bun x concurrently \"$(TSGO) -w -p tsconfig.main.json\" \"bun x tsc-alias -w -p tsconfig.main.json\"" \
 		"bun x esbuild src/cli/api.ts --bundle --format=esm --platform=node --target=node20 --outfile=dist/cli/api.mjs --external:zod --external:commander --external:@trpc/server --watch" \
 		"bun x nodemon --watch dist/cli/index.js --watch dist/cli/server.js --delay 500ms --exec 'NODE_ENV=development node dist/cli/index.js server --host $(or $(BACKEND_HOST),localhost) --port $(or $(BACKEND_PORT),3000)'" \
@@ -173,7 +173,7 @@ endif
 
 
 start: node_modules/.installed build-main build-preload build-static ## Build and start Electron app
-	@NODE_ENV=development MUX_DISABLE_TELEMETRY=$(or $(MUX_DISABLE_TELEMETRY),1) bunx electron --remote-debugging-port=9222 .
+	@NODE_ENV=development MUX_DISABLE_TELEMETRY=$(if $(MUX_ENABLE_TELEMETRY_IN_DEV),,$(or $(MUX_DISABLE_TELEMETRY),1)) bunx electron --remote-debugging-port=9222 .
 
 ## Build targets (can run in parallel)
 build: node_modules/.installed src/version.ts build-renderer build-main build-preload build-icons build-static ## Build all targets

src/browser/components/Settings/sections/ExperimentsSection.tsx

@@ -1,13 +1,13 @@
 import React, { useCallback, useMemo } from "react";
 import { useExperiment, useRemoteExperimentValue } from "@/browser/contexts/ExperimentsContext";
 import {
-  EXPERIMENTS,
   getExperimentList,
   EXPERIMENT_IDS,
   type ExperimentId,
 } from "@/common/constants/experiments";
 import { Switch } from "@/browser/components/ui/switch";
 import { useWorkspaceContext } from "@/browser/contexts/WorkspaceContext";
+import { useTelemetry } from "@/browser/hooks/useTelemetry";
 
 interface ExperimentRowProps {
   experimentId: ExperimentId;
@@ -17,41 +17,29 @@ interface ExperimentRowProps {
 }
 
 function ExperimentRow(props: ExperimentRowProps) {
-  const experiment = EXPERIMENTS[props.experimentId];
   const [enabled, setEnabled] = useExperiment(props.experimentId);
   const remote = useRemoteExperimentValue(props.experimentId);
-  const isRemoteControlled = remote ? remote.source !== "disabled" : false;
-  const canOverride = experiment.userOverridable === true;
-  const { onToggle } = props;
+  const telemetry = useTelemetry();
+  const { onToggle, experimentId } = props;
 
   const handleToggle = useCallback(
     (value: boolean) => {
-      // Allow toggle if not remote-controlled OR if user can override
-      if (isRemoteControlled && !canOverride) {
-        return;
-      }
-
       setEnabled(value);
+      // Track the override for analytics
+      telemetry.experimentOverridden(experimentId, remote?.value ?? null, value);
       onToggle?.(value);
     },
-    [isRemoteControlled, canOverride, setEnabled, onToggle]
+    [setEnabled, telemetry, experimentId, remote?.value, onToggle]
   );
 
   return (
     <div className="flex items-center justify-between py-3">
       <div className="flex-1 pr-4">
         <div className="text-foreground text-sm font-medium">{props.name}</div>
         <div className="text-muted mt-0.5 text-xs">{props.description}</div>
-        {isRemoteControlled ? (
-          <div className="text-muted mt-0.5 text-xs">
-            PostHog: {String(remote?.value ?? "loading")} ({remote?.source})
-            {canOverride ? " • overridable" : null}
-          </div>
-        ) : null}
       </div>
       <Switch
         checked={enabled}
-        disabled={isRemoteControlled && !canOverride}
         onCheckedChange={handleToggle}
         aria-label={`Toggle ${props.name}`}
       />
@@ -63,9 +51,10 @@ export function ExperimentsSection() {
   const allExperiments = getExperimentList();
   const { refreshWorkspaceMetadata } = useWorkspaceContext();
 
-  // Filter to only show experiments where showInSettings !== false
+  // Only show user-overridable experiments (non-overridable ones are hidden since users can't change them)
   const experiments = useMemo(
-    () => allExperiments.filter((exp) => exp.showInSettings !== false),
+    () =>
+      allExperiments.filter((exp) => exp.showInSettings !== false && exp.userOverridable === true),
     [allExperiments]
   );
 

src/browser/hooks/useTelemetry.ts

@@ -8,6 +8,7 @@ import {
   trackCommandUsed,
   trackVoiceTranscription,
   trackErrorOccurred,
+  trackExperimentOverridden,
 } from "@/common/telemetry";
 import type {
   ErrorContext,
@@ -35,6 +36,7 @@ import type {
  * telemetry.commandUsed(commandType);
  * telemetry.voiceTranscription(audioDurationSecs, success);
  * telemetry.errorOccurred(errorType, context);
+ * telemetry.experimentOverridden(experimentId, assignedVariant, userChoice);
  * ```
  */
 export function useTelemetry() {
@@ -83,6 +85,13 @@ export function useTelemetry() {
     trackErrorOccurred(errorType, context);
   }, []);
 
+  const experimentOverridden = useCallback(
+    (experimentId: string, assignedVariant: string | boolean | null, userChoice: boolean) => {
+      trackExperimentOverridden(experimentId, assignedVariant, userChoice);
+    },
+    []
+  );
+
   return {
     workspaceSwitched,
     workspaceCreated,
@@ -92,5 +101,6 @@ export function useTelemetry() {
     commandUsed,
     voiceTranscription,
     errorOccurred,
+    experimentOverridden,
   };
 }

src/common/orpc/schemas/telemetry.ts

@@ -97,6 +97,12 @@ const ErrorOccurredPropertiesSchema = z.object({
   context: ErrorContextSchema,
 });
 
+const ExperimentOverriddenPropertiesSchema = z.object({
+  experimentId: z.string(),
+  assignedVariant: z.union([z.string(), z.boolean(), z.null()]),
+  userChoice: z.boolean(),
+});
+
 // Union of all telemetry events
 export const TelemetryEventSchema = z.discriminatedUnion("event", [
   z.object({
@@ -135,6 +141,10 @@ export const TelemetryEventSchema = z.discriminatedUnion("event", [
     event: z.literal("error_occurred"),
     properties: ErrorOccurredPropertiesSchema,
   }),
+  z.object({
+    event: z.literal("experiment_overridden"),
+    properties: ExperimentOverriddenPropertiesSchema,
+  }),
 ]);
 
 // API schemas - only track endpoint, enabled state controlled by env var

src/common/telemetry/index.ts

@@ -24,6 +24,7 @@ export {
   trackCommandUsed,
   trackVoiceTranscription,
   trackErrorOccurred,
+  trackExperimentOverridden,
 } from "./tracking";
 
 // Utility for converting RuntimeConfig to telemetry-safe runtime type

src/common/telemetry/payload.ts

@@ -188,6 +188,19 @@ export interface ErrorOccurredPayload {
   context: ErrorContext;
 }
 
+/**
+ * Experiment override event - tracks when users manually toggle experiments
+ * This helps measure opt-out rates and understand user preferences
+ */
+export interface ExperimentOverriddenPayload {
+  /** Experiment identifier (e.g., 'post-compaction-context') */
+  experimentId: string;
+  /** The variant PostHog assigned (null if not remote-controlled) */
+  assignedVariant: string | boolean | null;
+  /** What the user chose (true = enabled, false = disabled) */
+  userChoice: boolean;
+}
+
 /**
  * Union type of all telemetry event payloads
  * Frontend sends these; backend adds BaseTelemetryProperties before forwarding to PostHog
@@ -201,4 +214,5 @@ export type TelemetryEventPayload =
   | { event: "provider_configured"; properties: ProviderConfiguredPayload }
   | { event: "command_used"; properties: CommandUsedPayload }
   | { event: "voice_transcription"; properties: VoiceTranscriptionPayload }
-  | { event: "error_occurred"; properties: ErrorOccurredPayload };
+  | { event: "error_occurred"; properties: ErrorOccurredPayload }
+  | { event: "experiment_overridden"; properties: ExperimentOverriddenPayload };

src/common/telemetry/tracking.ts

@@ -162,3 +162,20 @@ export function trackErrorOccurred(
     properties: { errorType, context },
   });
 }
+
+/**
+ * Track experiment override - when a user manually toggles an experiment
+ * @param experimentId - The experiment identifier
+ * @param assignedVariant - What PostHog assigned (null if not remote-controlled)
+ * @param userChoice - What the user chose (true = enabled, false = disabled)
+ */
+export function trackExperimentOverridden(
+  experimentId: string,
+  assignedVariant: string | boolean | null,
+  userChoice: boolean
+): void {
+  trackEvent({
+    event: "experiment_overridden",
+    properties: { experimentId, assignedVariant, userChoice },
+  });
+}