feat(pii): publish PII image to GHCR and add Presidio sidecar to Helm chart (#5188)

* feat(pii): publish PII image to GHCR and add Presidio sidecar to Helm chart

* fix(pii): allow app→PII NetworkPolicy egress, global tolerations, topology spread

Author: TheodoreSpeaks

.github/workflows/ci.yml

@@ -155,9 +155,8 @@ jobs:
           - dockerfile: ./docker/realtime.Dockerfile
             ghcr_image: ghcr.io/simstudioai/realtime
             ecr_repo_secret: ECR_REALTIME
-          # pii is ECR-only (private ECS sidecar) — no ghcr_image, so the tag
-          # step below skips GHCR for it.
           - dockerfile: ./docker/pii.Dockerfile
+            ghcr_image: ghcr.io/simstudioai/pii
             ecr_repo_secret: ECR_PII
     steps:
       - name: Checkout code
@@ -257,6 +256,8 @@ jobs:
             image: ghcr.io/simstudioai/migrations
           - dockerfile: ./docker/realtime.Dockerfile
             image: ghcr.io/simstudioai/realtime
+          - dockerfile: ./docker/pii.Dockerfile
+            image: ghcr.io/simstudioai/pii
 
     steps:
       - name: Checkout code
@@ -312,6 +313,7 @@ jobs:
           - image: ghcr.io/simstudioai/simstudio
           - image: ghcr.io/simstudioai/migrations
           - image: ghcr.io/simstudioai/realtime
+          - image: ghcr.io/simstudioai/pii
 
     steps:
       - name: Login to GHCR

.github/workflows/images.yml

@@ -1,5 +1,5 @@
 # ========================================
-# Combined Presidio service (analyzer + anonymizer) on a single port (3000)
+# Combined Presidio service (analyzer + anonymizer) on a single port (5001)
 # ========================================
 FROM python:3.12-slim-bookworm AS base
 

docker/pii.Dockerfile

@@ -48,6 +48,7 @@ Optional components (off by default):
 
 * **`copilot`** — the Sim Copilot service plus its own Postgres StatefulSet.
 * **`ollama`** — local LLM inference, with optional NVIDIA GPU support.
+* **`pii`** — Presidio PII redaction sidecar (analyzer + anonymizer) for the Guardrails PII block and log redaction. See [PII redaction](#pii-redaction).
 * **`telemetry`** — OpenTelemetry Collector wired to Jaeger / Prometheus / OTLP backends.
 * **`ingress`** — NGINX-style Ingress for the app and realtime services.
 * **`networkPolicy`** — east-west and egress isolation (blocks cloud metadata endpoints by default).
@@ -321,7 +322,7 @@ User-supplied `securityContext` values are merged with the defaults — your val
 Other security features:
 
 * `automountServiceAccountToken: false` on the ServiceAccount **and** every pod.
-* Every value in `app.env` and `realtime.env` is written to a chart-managed Secret and mounted via `envFrom: secretRef` — no values are inlined on the container spec. This eliminates a sensitivity classifier (no static list of "secret" keys to maintain) and ensures new provider keys can never accidentally leak into pod manifests. Two categories are inlined on the container instead: chart-computed values (`DATABASE_URL`, `SOCKET_SERVER_URL`, `OLLAMA_URL`) and operational defaults under `app.envDefaults` / `realtime.envDefaults` (rate limits, timeouts, IVM tunables, feature-flag defaults, branding defaults, `http://localhost:3000` URL fallbacks). Operational defaults are non-sensitive by design — moving them out of `app.env` keeps the Secret small and means External Secrets Operator users only have to map the keys they actually set, not every chart default. A value placed in `app.env` always wins over the same key in `app.envDefaults` (the template skips the inline default when an override exists).
+* Every value in `app.env` and `realtime.env` is written to a chart-managed Secret and mounted via `envFrom: secretRef` — no values are inlined on the container spec. This eliminates a sensitivity classifier (no static list of "secret" keys to maintain) and ensures new provider keys can never accidentally leak into pod manifests. Two categories are inlined on the container instead: chart-computed values (`DATABASE_URL`, `SOCKET_SERVER_URL`, `OLLAMA_URL`, `PII_URL`) and operational defaults under `app.envDefaults` / `realtime.envDefaults` (rate limits, timeouts, IVM tunables, feature-flag defaults, branding defaults, `http://localhost:3000` URL fallbacks). Operational defaults are non-sensitive by design — moving them out of `app.env` keeps the Secret small and means External Secrets Operator users only have to map the keys they actually set, not every chart default. A value placed in `app.env` always wins over the same key in `app.envDefaults` (the template skips the inline default when an override exists).
 * Optional `networkPolicy.enabled=true` enforces east-west isolation and blocks cloud metadata endpoints in egress.
 
 ---
@@ -354,6 +355,35 @@ Requires the Prometheus Operator CRDs. Scrapes `/metrics` on the app and realtim
 
 ---
 
+## PII redaction
+
+Sim can redact personally identifiable information using a [Presidio](https://microsoft.github.io/presidio/) sidecar (analyzer + anonymizer combined into one image listening on port 5001). Enable it with:
+
+```yaml
+pii:
+  enabled: true
+```
+
+When enabled, the chart deploys the sidecar (`<release>-pii` Deployment + Service) and **auto-wires** `PII_URL` on the app to the in-cluster service. The sidecar bundles five large spaCy models (en/es/it/pl/fi, ~2.2GB), so the first start takes ~3 minutes while models load — the `startupProbe` allows for this. Size the `pii.resources` for at least ~4Gi memory.
+
+This alone powers the **Guardrails PII block** and on-demand masking. To additionally turn on **automatic log redaction** (the org/workspace data-retention scrub), you must:
+
+```yaml
+app:
+  env:
+    PII_REDACTION: "true"
+    # The log-redaction path calls the app's own /api/guardrails/mask-batch,
+    # which must be reachable from inside the cluster. Set this to the in-cluster
+    # app Service URL (NOT the public ingress, which usually isn't hairpin-reachable).
+    INTERNAL_API_BASE_URL: "http://<release>-app.<namespace>.svc.cluster.local:3000"
+```
+
+Without a cluster-reachable `INTERNAL_API_BASE_URL` (it falls back to `NEXT_PUBLIC_APP_URL`), the redaction path fails closed — it scrubs affected fields to `[REDACTION_FAILED]` rather than leaking, but redaction won't actually run.
+
+> The PII image is published at `ghcr.io/simstudioai/pii` (multi-arch). If you mirror images into a private registry, retag it alongside the app/realtime/migrations images.
+
+---
+
 ## Troubleshooting
 
 ### `Error: execution error at (sim/templates/...): app.env.BETTER_AUTH_SECRET is required for production deployment`

helm/sim/README.md

@@ -16,6 +16,9 @@ Your release is named {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
 {{- if .Values.copilot.enabled }}
      kubectl --namespace {{ .Release.Namespace }} rollout status deployment/{{ include "sim.fullname" . }}-copilot
 {{- end }}
+{{- if .Values.pii.enabled }}
+     kubectl --namespace {{ .Release.Namespace }} rollout status deployment/{{ include "sim.fullname" . }}-pii
+{{- end }}
 
 2. Reach the application:
 {{- if and .Values.ingress.enabled .Values.ingress.app.host }}

helm/sim/templates/NOTES.txt

@@ -117,6 +117,22 @@ Ollama selector labels
 app.kubernetes.io/component: ollama
 {{- end }}
 
+{{/*
+PII (Presidio) specific labels
+*/}}
+{{- define "sim.pii.labels" -}}
+{{ include "sim.labels" . }}
+app.kubernetes.io/component: pii
+{{- end }}
+
+{{/*
+PII (Presidio) selector labels
+*/}}
+{{- define "sim.pii.selectorLabels" -}}
+{{ include "sim.selectorLabels" . }}
+app.kubernetes.io/component: pii
+{{- end }}
+
 {{/*
 Migrations specific labels
 */}}
@@ -261,8 +277,8 @@ externalSecrets.remoteRefs.app when ESO is enabled. When ESO is on, the
 chart-managed Secret is not rendered — anything not mapped via ESO would
 be silently missing at runtime.
 
-Chart-computed keys (DATABASE_URL, SOCKET_SERVER_URL, OLLAMA_URL) are
-exempt because they're inlined on the container, not sourced from the
+Chart-computed keys (DATABASE_URL, SOCKET_SERVER_URL, OLLAMA_URL, PII_URL)
+are exempt because they're inlined on the container, not sourced from the
 Secret.
 
 Fail-fast is only safe for ESO because we can introspect remoteRefs at
@@ -273,7 +289,7 @@ than enforced.
 {{- define "sim.validateExternalSecretCoverage" -}}
 {{- if and .Values.externalSecrets .Values.externalSecrets.enabled -}}
 {{- $remoteRefs := default (dict) (default (dict) .Values.externalSecrets.remoteRefs).app -}}
-{{- $chartComputed := list "DATABASE_URL" "SOCKET_SERVER_URL" "OLLAMA_URL" -}}
+{{- $chartComputed := list "DATABASE_URL" "SOCKET_SERVER_URL" "OLLAMA_URL" "PII_URL" -}}
 {{- $appEnv := default (dict) .Values.app.env -}}
 {{/*
   Required-key coverage: these are non-optional at runtime. With ESO enabled
@@ -430,6 +446,19 @@ Ollama URL
 {{- end }}
 {{- end }}
 
+{{/*
+PII (Presidio) sidecar URL
+*/}}
+{{- define "sim.piiUrl" -}}
+{{- if .Values.pii.enabled }}
+{{- $serviceName := printf "%s-pii" (include "sim.fullname" .) }}
+{{- $port := .Values.pii.service.port }}
+{{- printf "http://%s:%v" $serviceName $port }}
+{{- else }}
+{{- .Values.app.env.PII_URL | default "http://localhost:5001" }}
+{{- end }}
+{{- end }}
+
 {{/*
 Socket Server URL (internal)
 */}}

helm/sim/templates/_helpers.tpl

@@ -84,6 +84,8 @@ spec:
               value: {{ include "sim.socketServerUrl" . | quote }}
             - name: OLLAMA_URL
               value: {{ include "sim.ollamaUrl" . | quote }}
+            - name: PII_URL
+              value: {{ include "sim.piiUrl" . | quote }}
             {{- /*
               Skip envDefaults keys that the user has explicitly overridden in app.env
               with a non-empty value. K8s `env` takes precedence over `envFrom`, so an
@@ -112,7 +114,7 @@ spec:
               and in inline mode (values flow through the chart-managed Secret).
             */}}
             {{- if and .Values.app.secrets.existingSecret.enabled (not .Values.externalSecrets.enabled) }}
-            {{- $chartComputed := list "DATABASE_URL" "SOCKET_SERVER_URL" "OLLAMA_URL" }}
+            {{- $chartComputed := list "DATABASE_URL" "SOCKET_SERVER_URL" "OLLAMA_URL" "PII_URL" }}
             {{- range $key, $value := $appEnv }}
             {{- if and (ne (toString $value) "") (ne (toString $value) "<nil>") (not (has $key $chartComputed)) }}
             - name: {{ $key }}