From ef27443bbae8cd60755e74e7d60fb5965e1f6b98 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 24 Jul 2026 09:09:00 -0700 Subject: [PATCH 1/2] feat: Add a data importer for bringing cloud data into onPrem Currents support is able to create an export of Org data and make it available for import to onPrem --- charts/currents/templates/_common.tpl | 13 ++ charts/currents/templates/_helpers.tpl | 7 + .../templates/changestreams/deployment.yaml | 1 + charts/currents/templates/toolbox/pod.yaml | 98 ++++++++ charts/currents/templates/toolbox/pvc.yaml | 36 +++ charts/currents/values.yaml | 60 +++++ docs/README.md | 1 + docs/org-data-import.md | 220 ++++++++++++++++++ 8 files changed, 436 insertions(+) create mode 100644 charts/currents/templates/toolbox/pod.yaml create mode 100644 charts/currents/templates/toolbox/pvc.yaml create mode 100644 docs/org-data-import.md diff --git a/charts/currents/templates/_common.tpl b/charts/currents/templates/_common.tpl index 0b6368f..1e692dd 100644 --- a/charts/currents/templates/_common.tpl +++ b/charts/currents/templates/_common.tpl @@ -166,6 +166,19 @@ Create the name of the service account to use {{- end }} {{- end -}} +{{/* +ClickHouse restore mode. + +Suppresses change-stream-driven ClickHouse sync while an organization's +ClickHouse data is loaded from an external export (scripts/org-import). +*/}} +{{- define "currents.clickhouseRestoreModeEnv" -}} +{{- if (.Values.maintenance).clickhouseRestoreMode }} +- name: CURRENTS_CLICKHOUSE_RESTORE_MODE + value: "true" +{{- end }} +{{- end -}} + {{- define "currents.URLConfigEnv" -}} - name: GITLAB_REDIRECT_URL value: {{ printf "%s/integrations/gitlab/callback" (include "currents.url" (dict "context" . "input" .Values.currents.domains.recordApiHost)) }} diff --git a/charts/currents/templates/_helpers.tpl b/charts/currents/templates/_helpers.tpl index a181539..c198924 100644 --- a/charts/currents/templates/_helpers.tpl +++ b/charts/currents/templates/_helpers.tpl @@ -40,6 +40,13 @@ Create webhooks name and version as used by the chart label. {{- printf "%s-%s" (include "currents.fullname" .) .Values.webhooks.name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create toolbox name as used by the chart label. +*/}} +{{- define "currents.toolbox.fullname" -}} +{{- printf "%s-%s" (include "currents.fullname" .) .Values.toolbox.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- define "currents.url" -}} {{- if .context.Values.currents.domains.https -}} {{- printf "https://%s" .input -}} diff --git a/charts/currents/templates/changestreams/deployment.yaml b/charts/currents/templates/changestreams/deployment.yaml index 5409e10..b066dee 100644 --- a/charts/currents/templates/changestreams/deployment.yaml +++ b/charts/currents/templates/changestreams/deployment.yaml @@ -47,6 +47,7 @@ spec: value: "onprem" {{- include "currents.connectionConfigEnv" . | nindent 12 }} {{- include "currents.URLConfigEnv" . | nindent 12 }} + {{- include "currents.clickhouseRestoreModeEnv" . | nindent 12 }} {{- with (concat .Values.global.env .Values.changestreams.env) }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/currents/templates/toolbox/pod.yaml b/charts/currents/templates/toolbox/pod.yaml new file mode 100644 index 0000000..7d139a8 --- /dev/null +++ b/charts/currents/templates/toolbox/pod.yaml @@ -0,0 +1,98 @@ +{{- if .Values.toolbox.enabled }} +{{/* +Toolbox pod — a gated, manually-driven pod for the org import (ENG-912/913). + +Single container on the onPrem scheduler image, which now carries BOTH the org-import +CLIs (baked in at /app/packages/scheduler/dist/orgImport) and `mongorestore`. So the whole +import runs here with nothing kubectl-cp'd: the operator copies in a small download.json +and runs `currents-import`. + +It runs `sleep infinity` and does nothing on its own. +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "currents.toolbox.fullname" . }} + labels: + {{- include "currents.labels" (dict "context" . "component" .Values.toolbox.name) | nindent 4 }} + {{- with .Values.global.podAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + # A bare Pod, not a Job: the import is operator-driven and must never auto-retry. + restartPolicy: Never + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + serviceAccountName: {{ include "currents.serviceAccountName" . }} + {{- /* fsGroup makes the PVC writable by the non-root container user. */}} + {{- with .Values.global.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.global.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + containers: + - name: toolbox + {{- with .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + # The onPrem scheduler image: node + @currents/* + the org-import CLIs + mongorestore. + image: {{ include "currents.image" (dict "context" . "imageRoot" .Values.scheduler.image) }} + imagePullPolicy: {{ default .Values.global.imagePullPolicy .Values.scheduler.image.pullPolicy }} + # Override the image's pm2 entrypoint — nothing runs on its own here. + command: ["sleep", "infinity"] + env: + - name: CURRENTS_ENV + value: "onprem" + # node/mongorestore write config under $HOME; the root FS is read-only, so point it + # at the PVC. + - name: HOME + value: /data + {{- /* A bulk merge INSERT … SELECT into test_metric_v2 (+ its rollup MVs) can run for + minutes with no bytes on the socket; raise the client's request timeout past the + 30s default so it doesn't trip mid-insert. Matches ch-import's max_execution_time. */}} + - name: CLICKHOUSE_REQUEST_TIMEOUT_MS + value: {{ .Values.toolbox.clickhouseRequestTimeoutMs | default 3600000 | int64 | quote }} + {{- include "currents.connectionConfigEnv" . | nindent 8 }} + {{- /* The importer refuses a ClickHouse import unless it sees this — safe only + because the same helper sets it on the change-streams deployment. */}} + {{- include "currents.clickhouseRestoreModeEnv" . | nindent 8 }} + {{- with (concat .Values.global.env .Values.toolbox.env) }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /data + # Writable /tmp for the local merge FIFOs (the transformed BSON flows through the + # kernel pipe, not to disk). Needed because the root FS is read-only. + - name: tmp + mountPath: /tmp + {{- with .Values.toolbox.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "currents.toolbox.fullname" . }}-data + - name: tmp + emptyDir: + medium: Memory + {{- with .Values.toolbox.nodeSelector | default .Values.global.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.toolbox.tolerations | default .Values.global.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.toolbox.affinity | default .Values.global.affinity }} + affinity: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/currents/templates/toolbox/pvc.yaml b/charts/currents/templates/toolbox/pvc.yaml new file mode 100644 index 0000000..6ec9bc9 --- /dev/null +++ b/charts/currents/templates/toolbox/pvc.yaml @@ -0,0 +1,36 @@ +{{- if .Values.toolbox.enabled }} +{{/* +Scratch space for the toolbox pod. + +For an org import this holds the export artifact (*.bson, *.native, +manifest.json) plus the exactly-once import state file, so size it at roughly +1.5x the export's manifest.totals.exportedBytes + clickhouseTotals.exportedBytes. + +It deliberately outlives individual exec sessions: the ClickHouse import records +which tables it has already inserted in /data/.import-state.json, and that record +is what prevents a re-run from double-counting the AggregatingMergeTree rollups. +Deleting this volume mid-import destroys that guard. +*/}} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ include "currents.toolbox.fullname" . }}-data + labels: + {{- include "currents.labels" (dict "context" . "component" .Values.toolbox.name) | nindent 4 }} + annotations: + # Keep the volume (and the import state file) if the release is uninstalled + # while an import is in flight. + helm.sh/resource-policy: keep +spec: + accessModes: + - {{ .Values.toolbox.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.toolbox.persistence.size | quote }} +{{- if .Values.toolbox.persistence.volumeName }} + volumeName: {{ .Values.toolbox.persistence.volumeName }} +{{- end }} +{{- if .Values.toolbox.persistence.storageClass }} + storageClassName: "{{ .Values.toolbox.persistence.storageClass }}" +{{- end }} +{{- end }} diff --git a/charts/currents/values.yaml b/charts/currents/values.yaml index b35ac33..1922080 100644 --- a/charts/currents/values.yaml +++ b/charts/currents/values.yaml @@ -665,6 +665,43 @@ webhooks: affinity: {} # END Webhooks Configuration +# Toolbox Configuration +# An optional, manually-driven pod for one-off operational work (data import, +# ad-hoc maintenance). It runs `sleep infinity` and does nothing on its own — +# an operator execs in and drives it, then disables it again. +toolbox: + # -- Create the toolbox pod and its scratch PVC. Leave disabled for normal + # installs — with this off, nothing in this section renders. + enabled: false + name: toolbox + # -- Scratch space for artifacts and the import state file. Size it at ~1.5x + # the export's total bytes (manifest totals.exportedBytes + + # clickhouseTotals.exportedBytes). + # @default -- See [values.yaml] for default values + persistence: + accessMode: ReadWriteOnce + size: 20Gi + storageClass: "" + ## if volumeName is set, use this existing PersistentVolume + # volumeName: + # -- Additional environment variables for both toolbox containers. + env: [] + # -- ClickHouse client per-request socket timeout (ms) for the import. + clickhouseRequestTimeoutMs: 3600000 + # -- Resource limits for the toolbox containers. A large import is IO-bound; + # give it enough memory to stream comfortably. + resources: {} + # -- [Node selector] for the toolbox pod + # @default -- `{}` (defaults to global.nodeSelector) + nodeSelector: {} + # -- [Tolerations] for use with node taints + # @default -- `[]` (defaults to global.tolerations) + tolerations: [] + # -- Assign custom [affinity] rules to the pod + # @default -- `{}` (defaults to the global.affinity preset) + affinity: {} +# END Toolbox Configuration + # Service Account Configuration # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ serviceAccount: @@ -709,3 +746,26 @@ redis: sysctl: resourcesPreset: "none" # END Redis Configuration + +# Maintenance Configuration +# Special-purpose switches for one-off operational procedures. Everything here is +# expected to be OFF during normal operation — turn it on for the duration of a +# procedure, then turn it back off. +maintenance: + # -- Suppress change-stream-driven ClickHouse sync while an organization's + # ClickHouse data is restored from an external export (see scripts/org-import + # in the currents repo). Enable it together with `toolbox.enabled` for a + # Mongo + ClickHouse restore, then turn both off again. + # + # Without it, restoring documents into Mongo makes change-streams re-derive + # ClickHouse rows on top of the ones the import writes directly, permanently + # double-counting the hourly materialized views. + # + # Leave this OFF for a Mongo-only restore: there, change-streams re-deriving + # the restored documents is exactly how ClickHouse gets populated. + # + # NOTE: while this is on, NO org gets new ClickHouse metrics — the install + # keeps recording to Mongo while dashboards quietly stop updating. It is not + # a setting to leave enabled. + clickhouseRestoreMode: false +# END Maintenance Configuration diff --git a/docs/README.md b/docs/README.md index babfcc6..1ee8b4b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,7 @@ The Currents Helm Chart is stateless, so depends on being connected to stateful - [🚀 Start Here: EKS Quickstart](./eks/quickstart.md) - [EKS Upgrade Guide](./eks/upgrading.md) +- [Importing Organization Data (Cloud → Self-Hosted Migration)](./org-data-import.md) - [Development Guide](./developer-guide/README.md) - [Support Policy](./support.md) - [Configuration Reference](configuration.md) diff --git a/docs/org-data-import.md b/docs/org-data-import.md new file mode 100644 index 0000000..5e63cb9 --- /dev/null +++ b/docs/org-data-import.md @@ -0,0 +1,220 @@ +# Importing Organization Data (Cloud → Self-Hosted Migration) + +When you migrate an organization from Currents Cloud to your self-hosted install, +Currents support runs a one-off export of that org's data and sends you a single +small file, **`download.json`**, over a secure channel (usually Slack). + +This guide is everything you do on **your** side to load that data. The whole +import runs inside a temporary, gated **toolbox pod** in your cluster with one +command — `currents-import` — which downloads the artifact, restores it into your +MongoDB and ClickHouse, and verifies the result. + +## Before you start + +You need: + +- The **`download.json`** file from Currents support. +- The **mode**, which support will tell you: + - **`fresh`** — the organization does **not** exist on your install yet. The + data is imported under its original organization id. + - **`merge`** — the organization **already exists** on your install and you want + the cloud history folded into it. You will also need the **target + organization id** (24-character hex) on your install; support provides it. + - **`incremental`** — a follow-up top-up into an **already-migrated** org + (typically the final sweep at cutover), applied from a smaller Mongo-only + delta file. Same target organization id. See + [Follow-up (incremental) imports](#follow-up-incremental-imports). +- `kubectl` and `helm` access to the namespace where Currents is installed. +- Enough free space on the toolbox volume: roughly **1.5×** the export's total + size. Support can tell you the artifact size; set it with + `toolbox.persistence.size` (see step 1). + +The commands below use `` for your Helm release name and `` for its +namespace. Set a shell variable for the pod once it exists: + +```bash +POD=$(kubectl -n get pod -l app.kubernetes.io/component=toolbox -o name) +``` + +## How it works + +The import is a single orchestrated command that runs, in order: + +**fetch** the artifact (resumable, checksum-verified) → **preflight** safety +checks → **restore MongoDB** → **verify MongoDB** → **restore ClickHouse** → +**verify ClickHouse**. + +While it runs, Currents' change-streams service is put into a **maintenance +(restore) mode** so it does not re-derive and double-count the ClickHouse rollups +from the data being restored. You turn that on in step 1 and off in step 4. + +--- + +## Step 1 — Put the chart into import mode + +This enables the toolbox pod and puts change-streams into restore mode: + +```bash +helm upgrade currents/currents --reuse-values \ + --set toolbox.enabled=true \ + --set maintenance.clickhouseRestoreMode=true +``` + +If the artifact is large, also size the toolbox volume (≈1.5× the export): + +```bash + --set toolbox.persistence.size=50Gi +``` + +Wait for change-streams to roll out with restore mode active, and confirm it in +the logs: + +```bash +kubectl -n rollout status deploy/-currents-change-streams +kubectl -n logs deploy/-currents-change-streams | grep -i restore +# → "CURRENTS_CLICKHOUSE_RESTORE_MODE is ON — change-stream-driven ClickHouse sync is SUPPRESSED" +``` + +> **Do not skip the log check.** The importer refuses to touch ClickHouse unless +> restore mode is confirmed active — this is what protects your rollup totals. + +## Step 2 — Copy in the download file + +```bash +POD=$(kubectl get pod -l app.kubernetes.io/component=toolbox -o name) +kubectl cp download.json ${POD#pod/}:/data/download.json +``` + +`download.json` is the only file you copy in. It carries time-limited download +links; if the links have expired, ask support to re-sign it (that does not change +these steps). + +## Step 3 — Run the import + +**Fresh** (new organization): + +```bash +kubectl exec ${POD#pod/} -- \ + currents-import --mode=fresh --download=/data/download.json +``` + +**Merge** (into an existing organization — note the target org id): + +```bash +kubectl exec ${POD#pod/} -- \ + currents-import --mode=merge --targetOrgId=<24-hex-target-org> --download=/data/download.json +``` + +It streams progress to the logs — per collection and per ClickHouse table — and +finishes with `import complete`. Depending on size this takes from a few minutes +to a few hours. If it stops with an error, **do not disable import mode yet** — +see [Troubleshooting](#troubleshooting) to resume. + +## Step 4 — Return to normal operation + +Only after the import reports success: + +```bash +helm upgrade currents/currents --reuse-values \ + --set toolbox.enabled=false \ + --set maintenance.clickhouseRestoreMode=false +``` + +This removes the toolbox pod and takes change-streams out of restore mode. Confirm +the restore-mode log line is gone after the rollout, and the imported org's data +is visible in the app. + +--- + +## Follow-up (incremental) imports + +A migration is usually done in two passes: a large **initial** import (above), +done ahead of time, then a small **incremental** import at cutover that sweeps up +everything created in between — so the only "frozen" window is that final delta. + +Support sends you a second, smaller download file for the delta. The steps are +**identical** to the initial import (same chart flow, restore mode **on**) — just +use **`--mode=incremental`** with the **same target organization id**: + +```bash +kubectl -n exec ${POD#pod/} -- \ + currents-import --mode=incremental --targetOrgId=<24-hex-target-org> --download=/data/delta.json +``` + +The delta is imported and verified in full before the command returns, exactly like +the initial import. You can repeat an incremental import as many times as needed; +re-running is safe and will not double-count. + +> Support occasionally sends a **Mongo-only** delta (smaller download). Those import +> the same way, but ClickHouse for the delta repopulates in the **background** after +> the command returns — so keep restore mode on for a few minutes and confirm the new +> runs' charts have filled in before the final step. Support will tell you if a delta +> is Mongo-only, and can run a recovery step if anything is slow to appear. + +--- + +## Troubleshooting + +### The import is safe to resume — do not start over + +Progress is tracked on the toolbox volume (`/data/export/.import-state.json`), and +the download is resumable and checksum-verified. Re-running after a failure +continues where it left off rather than duplicating data. **Leave import mode on** +(step 1) until the import fully succeeds. + +### The ClickHouse step timed out or failed + +The largest ClickHouse table (`test_metric_v2`) is the most likely place to stall +on a big org. The MongoDB restore has already completed and been verified at this +point, so **resume just the ClickHouse import** — it skips tables that already +loaded and re-runs only what's missing: + +```bash +# merge: +kubectl exec ${POD#pod/} -- \ + node /app/packages/scheduler/dist/orgImport/cli.js ch-import \ + --merge --targetOrgId=<24-hex-target-org> --dir=/data/export + +# fresh (use the source org id, shown in the import logs / manifest): +kubectlexec ${POD#pod/} -- \ + node /app/packages/scheduler/dist/orgImport/cli.js ch-import \ + --orgId=<24-hex-source-org> --dir=/data/export +``` + +This is safe to run repeatedly: completed tables are skipped, and the re-inserted +table only adds rows it hasn't already inserted, so the rollup totals stay correct +**without any manual rebuild**. + +### Re-running `currents-import` says the org already exists / project already exists + +That is an intentional safety check, not a failure to fight. Once the MongoDB +restore has completed, re-running the **whole** `currents-import` is refused so it +can't double-import. To finish a partially-completed import, resume the +**ClickHouse** step directly as shown above rather than re-running the full +command. + +### The download links have expired + +`download.json`'s links are valid for up to 7 days from when support generated +them. If `fetch` fails with an expired/forbidden error, ask support to **re-sign** +and send a new `download.json`, then repeat from step 2. The rest of the import is +unaffected. + +### The change-streams pod is restarting during the import + +Some restart activity is expected while a large restore floods the MongoDB oplog. +The service self-heals and resumes on its own. If it enters a persistent +`CrashLoopBackOff`, the MongoDB **oplog is too small** for the restore burst — the +oplog window collapses faster than change-streams can keep up. Increase the +replica set's oplog size (for the MongoDB Community Operator, set +`replication.oplogSizeMB` in `additionalMongodConfig`) and, if the volume can't +hold a larger oplog, grow it too. This does not affect the correctness of the +import; the change-streams pod recovers once restore mode is turned off in step 4. + +### Getting help + +If you're stuck, send Currents support: + +- the **mode** and (for merge) the **target org id** you used, +- the `currents-import` (or `ch-import`) **logs**, and +- the output of `kubectl get pods`. From 03602cbde1d9bc8c8ba82c3e1849edc44cc41523 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 24 Jul 2026 09:11:56 -0700 Subject: [PATCH 2/2] chore: update docs --- docs/configuration.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 0da9051..f37343f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -213,6 +213,15 @@ The following table lists the configurable parameters of the `currents` chart an | webhooks.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] | | webhooks.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints | | webhooks.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the deployment | +| toolbox.enabled | bool | `false` | Create the toolbox pod and its scratch PVC. Leave disabled for normal installs — with this off, nothing in this section renders. | +| toolbox.name | string | `"toolbox"` | | +| toolbox.persistence | object | See [values.yaml] for default values | Scratch space for artifacts and the import state file. Size it at ~1.5x the export's total bytes (manifest totals.exportedBytes + clickhouseTotals.exportedBytes). | +| toolbox.env | list | `[]` | Additional environment variables for both toolbox containers. | +| toolbox.clickhouseRequestTimeoutMs | int | `3600000` | ClickHouse client per-request socket timeout (ms) for the import. | +| toolbox.resources | object | `{}` | Resource limits for the toolbox containers. A large import is IO-bound; give it enough memory to stream comfortably. | +| toolbox.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the toolbox pod | +| toolbox.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints | +| toolbox.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the pod | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | | serviceAccount.name | string | If not set and create is true, a name is generated using the fullname template | The name of the service account to use. | | serviceAccount.annotations | object | `{}` | Optional additional annotations to add to the Service Account. Templates are allowed for both keys and values. | @@ -228,5 +237,6 @@ The following table lists the configurable parameters of the `currents` chart an | redis.metrics.resourcesPreset | string | `"none"` | | | redis.volumePermissions.resourcesPreset | string | `"none"` | | | redis.sysctl.resourcesPreset | string | `"none"` | | +| maintenance.clickhouseRestoreMode | bool | `false` | Suppress change-stream-driven ClickHouse sync while an organization's ClickHouse data is restored from an external export (see scripts/org-import in the currents repo). Enable it together with `toolbox.enabled` for a Mongo + ClickHouse restore, then turn both off again. Without it, restoring documents into Mongo makes change-streams re-derive ClickHouse rows on top of the ones the import writes directly, permanently double-counting the hourly materialized views. Leave this OFF for a Mongo-only restore: there, change-streams re-deriving the restored documents is exactly how ClickHouse gets populated. NOTE: while this is on, NO org gets new ClickHouse metrics — the install keeps recording to Mongo while dashboards quietly stop updating. It is not a setting to leave enabled. | [values.yaml]: https://github.com/currents-dev/helm-charts/blob/main/charts/currents/values.yaml \ No newline at end of file