From 399f2d586afd7b3c05a5fcc385fbc4d143ba47ca Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Thu, 13 Aug 2026 14:19:28 -0700 Subject: [PATCH] fix: bump version to 0.7.4 and update app version to 2026-07-26-003 - Updated image tag to 2026-07-26-003 in values.yaml and documentation. - Added new configuration for ClickHouse insert block size in values.yaml and documentation. - Updated toolbox pod template to include environment variable for ClickHouse insert block size. --- charts/currents/Chart.yaml | 4 +- charts/currents/templates/toolbox/pod.yaml | 6 ++ charts/currents/values.yaml | 8 ++- docs/configuration.md | 5 +- docs/org-data-import.md | 64 ++++++++++++++++++++-- 5 files changed, 77 insertions(+), 10 deletions(-) diff --git a/charts/currents/Chart.yaml b/charts/currents/Chart.yaml index 8e55cdb..d71f4b3 100644 --- a/charts/currents/Chart.yaml +++ b/charts/currents/Chart.yaml @@ -5,10 +5,10 @@ home: https://currents.dev type: application # The chart version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.7.3 +version: 0.7.4 # Version number of the application being deployed. # Versions are not expected to follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "2026-07-26-002" +appVersion: "2026-07-26-003" maintainers: - name: Currents-dev url: https://currents.dev diff --git a/charts/currents/templates/toolbox/pod.yaml b/charts/currents/templates/toolbox/pod.yaml index 7d139a8..aaa93c1 100644 --- a/charts/currents/templates/toolbox/pod.yaml +++ b/charts/currents/templates/toolbox/pod.yaml @@ -58,6 +58,12 @@ spec: 30s default so it doesn't trip mid-insert. Matches ch-import's max_execution_time. */}} - name: CLICKHOUSE_REQUEST_TIMEOUT_MS value: {{ .Values.toolbox.clickhouseRequestTimeoutMs | default 3600000 | int64 | quote }} + {{- /* Caps how many rows ClickHouse squashes into one inserted block. Every block is + aggregated by test_metric_v2's two materialized views, so an unset value (the + server's 1M-row default) peaks around 3.1 GiB on the ClickHouse server and OOMs + a small one. ch-import reads this; --insert-block-rows overrides it. */}} + - name: CLICKHOUSE_MIN_INSERT_BLOCK_SIZE_ROWS + value: {{ .Values.toolbox.clickhouseInsertBlockRows | default 65536 | int64 | quote }} {{- include "currents.connectionConfigEnv" . | nindent 8 }} {{- /* The importer refuses a ClickHouse import unless it sees this — safe only because the same helper sets it on the change-streams deployment. */}} diff --git a/charts/currents/values.yaml b/charts/currents/values.yaml index fc701e6..9d0dcc2 100644 --- a/charts/currents/values.yaml +++ b/charts/currents/values.yaml @@ -27,7 +27,7 @@ currents: # @section -- Frequently Used key: password # -- The image tag to use for the Currents images - imageTag: 2026-07-26-002 + imageTag: 2026-07-26-003 email: # -- Which transport to send outgoing email through: `smtp` or `ses`. # With `ses` the SMTP settings are ignored and no SMTP credentials are @@ -702,6 +702,12 @@ toolbox: env: [] # -- ClickHouse client per-request socket timeout (ms) for the import. clickhouseRequestTimeoutMs: 3600000 + # -- Rows per block the import inserts into ClickHouse. Each block is aggregated + # by `test_metric_v2`'s two materialized views, so this sets the insert's peak + # memory on the ClickHouse server: measured against the real schema, 1M rows (the + # ClickHouse default) peaks at ~3.1 GiB and 65536 at ~1.3 GiB. Lower it if + # ClickHouse has less than 8 GiB or an insert fails with MEMORY_LIMIT_EXCEEDED. + clickhouseInsertBlockRows: 65536 # -- Resource limits for the toolbox containers. A large import is IO-bound; # give it enough memory to stream comfortably. resources: {} diff --git a/docs/configuration.md b/docs/configuration.md index e826e0f..1d425be 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,6 +1,6 @@ # Configuration Reference -![Version: 0.7.3](https://img.shields.io/badge/Version-0.7.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2026-07-26-002](https://img.shields.io/badge/AppVersion-2026--07--26--002-informational?style=flat-square) +![Version: 0.7.4](https://img.shields.io/badge/Version-0.7.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2026-07-26-003](https://img.shields.io/badge/AppVersion-2026--07--26--003-informational?style=flat-square) ## Requirements @@ -88,7 +88,7 @@ The following table lists the configurable parameters of the `currents` chart an | Key | Type | Default | Description | |-----|------|---------|-------------| | currents.rootUser.email | string | `"admin@{{ .Values.currents.domains.appHost }}"` | The email address of the root user | -| currents.imageTag | string | `"2026-07-26-002"` | The image tag to use for the Currents images | +| currents.imageTag | string | `"2026-07-26-003"` | The image tag to use for the Currents images | | currents.email.transporter | string | `"smtp"` | Which transport to send outgoing email through: `smtp` or `ses`. With `ses` the SMTP settings are ignored and no SMTP credentials are needed — the AWS SDK resolves credentials from the pod itself, so grant the Currents service account permission to send. See [Using IAM Roles for Sending Email with SES](./eks/iam.md#using-iam-roles-for-sending-email-with-ses). | | currents.email.from | tpl/string | `""` | The email address to send from. Defaults to `currents.email.smtp.from` when unset, which is retained for compatibility. | | currents.email.ses.region | string | `""` | The AWS region to send through. Required when `transporter` is `ses`, and the `from` address must be a verified identity in that region. | @@ -221,6 +221,7 @@ The following table lists the configurable parameters of the `currents` chart an | toolbox.persistence | object | See [values.yaml] for default values | Scratch space for artifacts and the import state file. Size it at ~1.5x the export's total bytes (manifest totals.exportedBytes + clickhouseTotals.exportedBytes). | | toolbox.env | list | `[]` | Additional environment variables for toolbox containers. | | toolbox.clickhouseRequestTimeoutMs | int | `3600000` | ClickHouse client per-request socket timeout (ms) for the import. | +| toolbox.clickhouseInsertBlockRows | int | `65536` | Rows per block the import inserts into ClickHouse. Each block is aggregated by `test_metric_v2`'s two materialized views, so this sets the insert's peak memory on the ClickHouse server: measured against the real schema, 1M rows (the ClickHouse default) peaks at ~3.1 GiB and 65536 at ~1.3 GiB. Lower it if ClickHouse has less than 8 GiB or an insert fails with MEMORY_LIMIT_EXCEEDED. | | toolbox.resources | object | `{}` | Resource limits for the toolbox containers. A large import is IO-bound; give it enough memory to stream comfortably. | | toolbox.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the toolbox pod | | toolbox.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints | diff --git a/docs/org-data-import.md b/docs/org-data-import.md index 35dd3e5..ff5f217 100644 --- a/docs/org-data-import.md +++ b/docs/org-data-import.md @@ -28,6 +28,15 @@ You need: - Enough free space on the toolbox volume: roughly **1.5×** the export's total size. Support can tell you the artifact size; set it with `toolbox.persistence.size` (see step 1). +- **At least 8 GiB of memory for ClickHouse**, with a memory limit actually set on + it. The import's heaviest step inserts into ClickHouse's largest table, and each + inserted block is aggregated into the hourly rollups as it lands. The commands + below cap that block size at 65,536 rows, which keeps the insert's peak around + 1.3 GiB instead of the ~3.1 GiB ClickHouse's own default would use. + +> A ClickHouse pod with **no** memory limit is worse than one with a limit: with no +> limit it sizes itself against the whole node, never trips its own memory guard, and +> gets killed by the kernel mid-import instead of failing one query cleanly. The commands below use `` for your Helm release name and `` for its namespace. Set a shell variable for the pod once it exists: @@ -95,16 +104,25 @@ these steps). ```bash kubectl exec ${POD#pod/} -- \ - currents-import --mode=fresh --download=/data/download.json + currents-import --mode=fresh --download=/data/download.json \ + --insert-block-rows=65536 ``` **Merge** (into an existing organization — note the target org id): ```bash kubectl exec ${POD#pod/} -- \ - currents-import --mode=merge --targetOrgId=<24-hex-target-org> --download=/data/download.json + currents-import --mode=merge --targetOrgId=<24-hex-target-org> --download=/data/download.json \ + --insert-block-rows=65536 ``` +`--insert-block-rows` is the ClickHouse memory cap described in +[Before you start](#before-you-start). The chart already sets the same value on the +toolbox pod, so the flag is belt-and-braces — pass a **lower** number (halve it) if +ClickHouse has less than 8 GiB, and see +[the import failed with MEMORY_LIMIT_EXCEEDED](#the-clickhouse-step-failed-with-memory_limit_exceeded) +if an insert still runs out of memory. + It streams progress to the logs — per collection and per ClickHouse table — and finishes with `import complete`. Depending on size this takes from a few minutes to a few hours. If it stops with an error, **do not disable import mode yet** — @@ -138,7 +156,8 @@ use **`--mode=incremental`** with the **same target organization id**: ```bash kubectl -n exec ${POD#pod/} -- \ - currents-import --mode=incremental --targetOrgId=<24-hex-target-org> --download=/data/delta.json + currents-import --mode=incremental --targetOrgId=<24-hex-target-org> --download=/data/delta.json \ + --insert-block-rows=65536 ``` The delta is imported and verified in full before the command returns, exactly like @@ -173,18 +192,50 @@ loaded and re-runs only what's missing: # merge: kubectl exec ${POD#pod/} -- \ node /app/packages/scheduler/dist/orgImport/cli.js ch-import \ - --merge --targetOrgId=<24-hex-target-org> --dir=/data/export + --merge --targetOrgId=<24-hex-target-org> --dir=/data/export --insert-block-rows=65536 # fresh (use the source org id, shown in the import logs / manifest): kubectl exec ${POD#pod/} -- \ node /app/packages/scheduler/dist/orgImport/cli.js ch-import \ - --orgId=<24-hex-source-org> --dir=/data/export + --orgId=<24-hex-source-org> --dir=/data/export --insert-block-rows=65536 ``` This is safe to run repeatedly: completed tables are skipped, and the re-inserted table only adds rows it hasn't already inserted, so the rollup totals stay correct **without any manual rebuild**. +### The ClickHouse step failed with MEMORY_LIMIT_EXCEEDED + +``` +ClickHouseError: (total) memory limit exceeded: would use 21.74 GiB … + code: '241', type: 'MEMORY_LIMIT_EXCEEDED' +``` + +The first thing to try is a **smaller block size** — halve `--insert-block-rows` (to +`32768`, then `16384`) on the resume command above. From 65,536 downward each halving +roughly halves the insert's peak memory, and it costs no meaningful time. + +If halving it changes nothing — the insert fails at the **same** size no matter what +you pass — the block size is not what's running out of memory, and there is no value +low enough to fix it. On an **incremental** import, the other large allocation is the +check that skips rows already imported, either give ClickHouse more memory for the duration of the cutover, or ask +support to run a recovery step that loads that one table a different way. + +### The MongoDB restore reports "N document(s) failed to restore" + +Duplicate-key errors like this are **expected** when you re-run an import that already +restored MongoDB: + +``` +continuing through error: E11000 duplicate key error collection: currents.tests … +0 document(s) restored successfully. 1906389 document(s) failed to restore. +``` + +Every "failure" is a document that is **already** in your database, which is why the +verification step immediately after it passes. It is not data loss and it does not +need a retry. If the MongoDB verify step *fails*, that is a different problem — send +support the logs. + ### Re-running `currents-import` says the org already exists / project already exists That is an intentional safety check, not a failure to fight. Once the MongoDB @@ -218,3 +269,6 @@ If you're stuck, send Currents support: - the **mode** and (for merge) the **target org id** you used, - the `currents-import` (or `ch-import`) **logs**, and - the output of `kubectl get pods`. + +For anything ClickHouse-related, also include its **memory limit** and the +`--insert-block-rows` you passed — those two decide most import memory failures.