diff --git a/.agents/skills/authoring-ci-workflows/SKILL.md b/.agents/skills/authoring-ci-workflows/SKILL.md
index b2d3fdd940cb..1721bf6d0b76 100644
--- a/.agents/skills/authoring-ci-workflows/SKILL.md
+++ b/.agents/skills/authoring-ci-workflows/SKILL.md
@@ -9,6 +9,8 @@ description: >
# Authoring CI workflows
+Before you propose a change to CI, check [things already tried](../../../docs/internal/ci-things-already-tried.md) for the idea. It records what was measured, and why some good-sounding changes were reverted or rejected.
+
Conventions for `.github/workflows/**` and `.github/actions/**`.
The linters own the mechanical rules (below); this skill is the **judgment calls** they can't enforce.
diff --git a/.agents/skills/debugging-ci-failures/SKILL.md b/.agents/skills/debugging-ci-failures/SKILL.md
index 76e8b2eb0e0f..a6b3345383a6 100644
--- a/.agents/skills/debugging-ci-failures/SKILL.md
+++ b/.agents/skills/debugging-ci-failures/SKILL.md
@@ -15,6 +15,8 @@ description: >
# Debugging PostHog CI failures
+Before you propose a change to CI, check [things already tried](../../../docs/internal/ci-things-already-tried.md) for the idea. It records what was measured, and why some good-sounding changes were reverted or rejected.
+
Find the first meaningful failure, classify it, reproduce the smallest useful
case locally when appropriate, and report the result. Avoid public-visible or
irreversible actions unless the user explicitly asks.
diff --git a/.agents/skills/depot-container-builds/SKILL.md b/.agents/skills/depot-container-builds/SKILL.md
index 03b89fa41c81..ceac7567a308 100644
--- a/.agents/skills/depot-container-builds/SKILL.md
+++ b/.agents/skills/depot-container-builds/SKILL.md
@@ -12,6 +12,8 @@ description: >
# Depot Container Builds
+Before you propose a change to CI, check [things already tried](../../../docs/internal/ci-things-already-tried.md) for the idea. It records what was measured, and why some good-sounding changes were reverted or rejected.
+
Depot runs Docker image builds on remote high-performance builders (16 CPU, 32 GB RAM, NVMe SSD cache). `depot build` is a drop-in replacement for `docker build` / `docker buildx build`. `depot bake` replaces `docker buildx bake`.
## Project Selection for Multi-Org Users
diff --git a/.agents/skills/django-migrations/SKILL.md b/.agents/skills/django-migrations/SKILL.md
index 0ae53452ad12..59a844de10ff 100644
--- a/.agents/skills/django-migrations/SKILL.md
+++ b/.agents/skills/django-migrations/SKILL.md
@@ -5,6 +5,8 @@ description: Django migration patterns and safety workflow for PostHog. Use when
# Django migrations
+Before you propose a change to the migration history or to how migrations run, check [things already tried](../../../docs/internal/ci-things-already-tried.md). It records the closed attempts at squashing the history and at the Person table cutover.
+
Read these files first, before writing or editing a migration:
- `docs/published/handbook/engineering/developing-locally.md` (`## Django migrations`, `### Non-blocking migrations`, `### Resolving merge conflicts`)
diff --git a/.agents/skills/fixing-flaky-tests/SKILL.md b/.agents/skills/fixing-flaky-tests/SKILL.md
index 3ea897e79c2c..07fea3c611dc 100644
--- a/.agents/skills/fixing-flaky-tests/SKILL.md
+++ b/.agents/skills/fixing-flaky-tests/SKILL.md
@@ -9,6 +9,8 @@ description: >
# Fixing flaky tests
+Before you propose a change to how the suite runs in CI, check [things already tried](../../../docs/internal/ci-things-already-tried.md). It records measured verdicts on test parallelism, sharding, and coverage-based selection, so a rejected approach is not rebuilt.
+
Three non-negotiables, in order:
1. **Reproduce before you fix.** A fix for a failure you never observed is a guess.
diff --git a/.agents/skills/improving-drf-endpoints/SKILL.md b/.agents/skills/improving-drf-endpoints/SKILL.md
index 66a73d556e3e..b3e79013c04d 100644
--- a/.agents/skills/improving-drf-endpoints/SKILL.md
+++ b/.agents/skills/improving-drf-endpoints/SKILL.md
@@ -5,6 +5,8 @@ description: Use when editing, reviewing, or auditing DRF viewsets and serialize
# Improving DRF Endpoints
+Before you propose a contract test against the generated OpenAPI schema, check [things already tried](../../../docs/internal/ci-things-already-tried.md). Six PRs took that idea, and none merged.
+
## Overview
Serializer fields are the source of truth for PostHog's entire type pipeline:
diff --git a/.agents/skills/isolating-product-facade-contracts/SKILL.md b/.agents/skills/isolating-product-facade-contracts/SKILL.md
index 1f967eea5e99..8f0e73c45a78 100644
--- a/.agents/skills/isolating-product-facade-contracts/SKILL.md
+++ b/.agents/skills/isolating-product-facade-contracts/SKILL.md
@@ -5,6 +5,8 @@ description: Plan and execute product isolation migrations to a facade plus cont
# Isolating a product with facade and contracts
+Before you choose a product to isolate, check [things already tried](../../../docs/internal/ci-things-already-tried.md). It records which product was the first candidate, and why the field test moved to another one.
+
Use this skill to migrate an existing product to the isolated architecture used by Visual review.
Optimize for short calendar exposure, not small diffs: authoring is cheap and the verification
chain catches mechanical breakage, while human review latency and a fast-moving master are the
diff --git a/.agents/skills/maintaining-python-tests/SKILL.md b/.agents/skills/maintaining-python-tests/SKILL.md
index 38647c440a0b..628823755615 100644
--- a/.agents/skills/maintaining-python-tests/SKILL.md
+++ b/.agents/skills/maintaining-python-tests/SKILL.md
@@ -6,6 +6,8 @@ description: >
# Maintaining Python tests
+Before you propose a change to how the suite runs in CI, check [things already tried](../../../docs/internal/ci-things-already-tried.md). It records measured verdicts on test parallelism, sharding, and coverage-based selection, so a rejected approach is not rebuilt.
+
Use this skill for an existing Python test suite. Use `/writing-tests` before adding or substantially changing coverage. Use `/fixing-flaky-tests` when intermittent failure is the main problem.
The goal is not a smaller test count. The goal is a suite that catches the same realistic regressions with less compute, less waiting, and less maintenance.
diff --git a/.agents/skills/stacking-prs/SKILL.md b/.agents/skills/stacking-prs/SKILL.md
index 60d893b92a4f..c121ade23e5e 100644
--- a/.agents/skills/stacking-prs/SKILL.md
+++ b/.agents/skills/stacking-prs/SKILL.md
@@ -12,6 +12,8 @@ description: >
# Stacked PRs with `gh stack`
+Before you divide a change into layers, check [things already tried](../../../docs/internal/ci-things-already-tried.md). It records when a stack cost more review effort than one PR.
+
GitHub's native stacked PRs are enabled on this repo.
A stack is an ordered chain of PRs where each one targets the branch of the PR below it; the bottom PR targets `master`.
GitHub tracks the chain as a first-class object: the PR UI shows a stack map, branch protections (code owner approval, required checks) apply to **every** layer including mid-stack ones, and CI that runs on `master` PRs runs on every layer.
diff --git a/.depot/workflows/ci-backend-update-test-timing.yml b/.depot/workflows/ci-backend-update-test-timing.yml
index 7544672342e6..df868de18ea1 100644
--- a/.depot/workflows/ci-backend-update-test-timing.yml
+++ b/.depot/workflows/ci-backend-update-test-timing.yml
@@ -67,9 +67,14 @@ jobs:
posthog-test-durations-
- name: Set the previous map aside
+ # .test_durations is not tracked, so there is nothing to restore it from
+ # and a cache miss leaves no file at all.
run: |
- cp .test_durations /tmp/previous_durations
- git checkout -- .test_durations
+ if [ -f .test_durations ]; then
+ mv .test_durations /tmp/previous_durations
+ else
+ echo "::warning::No cached map restored; a drifting Products slice has no stand-in"
+ fi
- name: Download timing artifacts
env:
@@ -103,12 +108,21 @@ jobs:
set -e
if [ "$products_status" -ne 0 ]; then
echo "::warning::Products durations refresh exited $products_status; keeping the previous products slice. Drift or an incomplete JUnit set are the expected causes, see the log above."
- # Only entries the fresh slices do not carry: the outlier merge would
- # otherwise let a stale value here beat a fresh one for the same test.
- jq -s 'add' $(ls /tmp/core_durations /tmp/temporal_durations 2>/dev/null) > /tmp/fresh_durations
- jq --slurpfile fresh /tmp/fresh_durations \
- 'with_entries(select((.key | startswith("products/")) and ($fresh[0][.key] == null)))' \
- /tmp/previous_durations > /tmp/products_durations
+ if [ -f /tmp/previous_durations ]; then
+ # Only entries the fresh slices do not carry: the outlier merge would
+ # otherwise let a stale value here beat a fresh one for the same test.
+ jq -s 'add' $(ls /tmp/core_durations /tmp/temporal_durations 2>/dev/null) > /tmp/fresh_durations
+ jq --slurpfile fresh /tmp/fresh_durations \
+ 'with_entries(select((.key | startswith("products/")) and ($fresh[0][.key] == null)))' \
+ /tmp/previous_durations > /tmp/products_durations
+ else
+ # No cached slice to stand in and no fresh one. A map with no product
+ # entries would still become the newest cache and the latest successful
+ # artifact, so every consumer would fall back to the file-count estimate
+ # and under-shard the fixture-heavy products. Leave the older map newest.
+ echo "::error::Products slice failed with no previous map to stand in; refusing to publish a map without product entries"
+ exit 1
+ fi
fi
uv run .github/scripts/optimize_test_durations.py .test_durations \
--merge-files /tmp/core_durations /tmp/temporal_durations /tmp/products_durations \
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index 03b1b47157b8..d8e502b19a08 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -942,8 +942,15 @@ jobs:
- name: Run product tests
# --force: discover already decided this product needs testing, skip turbo cache
# --log-order=stream: stream pytest output live instead of buffering until completion
- # pytest_args: optional pytest-split flags for sharded products (e.g. "-- --splits 3 --group 1")
+ # legs: one turbo invocation per entry, each with its own pytest-split flags
+ # (e.g. "-- --splits 3 --group 1"). A job holds several legs when a split
+ # product's last shard had budget left over for whole small products, and
+ # pytest-split flags must not leak from that shard onto its job-mates.
env:
+ PRODUCT_LEGS: ${{ toJSON(matrix.legs) }}
+ # Read only when an unrebased branch planned the pre-legs shape.
+ LEGACY_FILTERS: ${{ matrix.filters }}
+ LEGACY_PYTEST_ARGS: ${{ matrix.pytest_args }}
CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
# products/tasks/backend/temporal moved here from the Django Temporal
# segment, and its conftest and workflows talk to Modal. Injected only
@@ -965,17 +972,42 @@ jobs:
# bin-packed buckets (each product writes its own file) and split products (unioned later).
# sysmon: Python 3.12+'s low-overhead coverage backend, ~a few % vs ~20% for the C tracer.
COVERAGE_CORE: sysmon
+ shell: bash
run: |
set +e
- pnpm turbo run backend:test ${{ matrix.filters }} --concurrency=1 --output-logs=full --force --log-order=stream ${{ matrix.pytest_args }}
- exit_code=$?
- set -e
- if [ $exit_code -eq 5 ]; then
- echo "No tests collected for this shard, this is expected when splitting tests"
- exit 0
- else
- exit $exit_code
+ overall=0
+ # A workflow edit reaches an open PR before the script edit does, so a
+ # branch that has not rebased still plans the flat {filters, pytest_args}
+ # shape. Normalize that into one leg rather than requiring the new key.
+ if [ -z "$PRODUCT_LEGS" ] || [ "$PRODUCT_LEGS" = "null" ]; then
+ PRODUCT_LEGS=$(jq -cn --arg f "$LEGACY_FILTERS" --arg a "$LEGACY_PYTEST_ARGS" \
+ '[{filters: $f, pytest_args: $a}]')
+ echo "Matrix entry carries no legs; running the legacy single invocation"
+ fi
+ leg_count=$(jq 'length' <<< "$PRODUCT_LEGS")
+ # Neither shape present means the job would pass having run no tests, and
+ # the rollup gate only reads this step's outcome. Fail loudly instead.
+ if [ -z "$leg_count" ] || [ "$leg_count" -lt 1 ]; then
+ echo "::error::No test legs in this matrix entry; refusing to pass without running tests"
+ exit 1
fi
+ for ((leg_index = 0; leg_index < leg_count; leg_index++)); do
+ leg_filters=$(jq -r ".[$leg_index].filters" <<< "$PRODUCT_LEGS")
+ leg_args=$(jq -r ".[$leg_index].pytest_args" <<< "$PRODUCT_LEGS")
+ echo "::group::turbo backend:test $leg_filters $leg_args"
+ # Word splitting is intended here: both variables carry several flags.
+ # shellcheck disable=SC2086
+ pnpm turbo run backend:test $leg_filters --concurrency=1 --output-logs=full --force --log-order=stream $leg_args
+ exit_code=$?
+ echo "::endgroup::"
+ if [ $exit_code -eq 5 ]; then
+ echo "No tests collected for this leg, this is expected when splitting tests"
+ elif [ $exit_code -ne 0 ]; then
+ overall=$exit_code
+ fi
+ done
+ set -e
+ exit $overall
# Lightweight repo-wide checks that only need Python + uv (no Docker/DB).
# Consolidates checks that previously each spun up their own runner.
repo-checks:
diff --git a/.github/scripts/turbo-discover-sizing.test.js b/.github/scripts/turbo-discover-sizing.test.js
index 557ee10e8cf8..6ba450e302be 100644
--- a/.github/scripts/turbo-discover-sizing.test.js
+++ b/.github/scripts/turbo-discover-sizing.test.js
@@ -7,7 +7,7 @@
const test = require('node:test')
const assert = require('node:assert/strict')
-const { pruneDeadDurations, getSegmentDuration, calculateShards, resolveProductSizing, buildMatrix, PRODUCT_JOB_OVERHEAD_SECONDS, PRODUCT_SAFETY_FACTOR, TARGET_WALL_SECONDS } = require('./turbo-discover.js')
+const { pruneDeadDurations, getSegmentDuration, calculateShards, resolveProductSizing, buildMatrix, productSplitShards, PRODUCT_JOB_OVERHEAD_SECONDS, TARGET_WALL_SECONDS } = require('./turbo-discover.js')
// A path that exists in every checkout, so the existence check is deterministic.
const LIVE_FILE = '.github/scripts/turbo-discover.js'
@@ -95,8 +95,8 @@ test('buildMatrix splits a product to the shared wall target', () => {
const matrix = buildMatrix(['big-one'], union, true)
- // 2000s of work, with the safety factor, over a (target - overhead) budget per shard.
- assert.equal(matrix.length, Math.ceil((2000 * PRODUCT_SAFETY_FACTOR) / (TARGET_WALL_SECONDS - O)))
+ // 2000s of work, with the imbalance margin, over a (target - overhead) budget.
+ assert.equal(matrix.length, productSplitShards({ work: 2000, heavyCount: 0, lightWork: 2000, maxLight: 50, testCount: 40 }))
assert.match(matrix[0].group, /^big-one \(1\/\d+\)$/)
})
@@ -107,5 +107,108 @@ test('buildMatrix leaves a small product packed', () => {
assert.equal(matrix.length, 1)
assert.equal(matrix[0].group, 'small-one')
+ assert.deepEqual(matrix[0].legs, [{ filters: '--filter=@posthog/products-small-one', pytest_args: '' }])
+})
+
+test('a single-invocation entry keeps the pre-legs keys for unrebased branches', () => {
+ // A workflow edit lands on an open PR before this script does, so an old
+ // workflow reading matrix.filters must still find something to run.
+ const union = { 'products/small_one/backend/test_c.py::test_c': 100 }
+
+ const matrix = buildMatrix(['small-one'], union, true)
+
+ assert.equal(matrix[0].filters, '--filter=@posthog/products-small-one')
assert.equal(matrix[0].pytest_args, '')
})
+
+test('productSplitShards sizes the worst chunk, not the mean', () => {
+ const budget = TARGET_WALL_SECONDS - PRODUCT_JOB_OVERHEAD_SECONDS
+ const evenly = { work: 1000, heavyCount: 0, lightWork: 1000, maxLight: 10, testCount: 100 }
+ const coarse = { work: 1000, heavyCount: 0, lightWork: 1000, maxLight: 150, testCount: 100 }
+
+ // Same total work; the coarser grain cannot be cut as finely, so it needs more
+ // shards to keep its worst chunk inside the budget.
+ assert.ok(productSplitShards(coarse) > productSplitShards(evenly))
+ assert.ok(evenly.lightWork / productSplitShards(evenly) + evenly.maxLight <= budget)
+ assert.ok(coarse.lightWork / productSplitShards(coarse) + coarse.maxLight <= budget)
+})
+
+test('tests above half the budget each hold a shard of their own', () => {
+ const budget = TARGET_WALL_SECONDS - PRODUCT_JOB_OVERHEAD_SECONDS
+ const heavy = Math.ceil(budget * 0.8)
+
+ // Ten tests this size cannot pair, so no count below ten holds the budget,
+ // however the total work divides.
+ assert.equal(productSplitShards({ work: heavy * 10, heavyCount: 10, lightWork: 0, maxLight: 0, testCount: 10 }), 10)
+ // One heavy test and a sliver stays bounded rather than asking for a shard per
+ // second of remainder. Two tests cannot fill three shards, so the count stops
+ // there rather than planning one that collects nothing.
+ assert.equal(
+ productSplitShards({ work: budget + 1, heavyCount: 1, lightWork: 1, maxLight: 1, testCount: 2 }),
+ 2
+ )
+})
+
+test('the count never exceeds the tests there are to place', () => {
+ // The fragmentation charge assumes a shape the suite may not have. Past the
+ // test count a shard collects nothing and spends a runner for it.
+ const many = { work: 10000, heavyCount: 3, lightWork: 100, maxLight: 10, testCount: 5 }
+
+ assert.equal(productSplitShards(many), 5)
+})
+
+test('a product holding one test is never split', () => {
+ const budget = TARGET_WALL_SECONDS - PRODUCT_JOB_OVERHEAD_SECONDS
+
+ // One test over the budget still gets one job: a second would collect nothing,
+ // and no split shortens the first.
+ assert.equal(
+ productSplitShards({ work: budget + 1, heavyCount: 1, lightWork: 0, maxLight: 0, testCount: 1 }),
+ 1
+ )
+})
+
+test('a heavy test between light ones splits the light run', () => {
+ // Ordered [53, 301, 133] against a 320s budget: either contiguous cut leaves a
+ // 354s or 434s chunk, so two shards cannot hold the budget however it is cut.
+ assert.equal(productSplitShards({ work: 487, heavyCount: 1, lightWork: 186, maxLight: 133, testCount: 3 }), 3)
+})
+
+test('a product that fits one shard is packed, not split by its own margin', () => {
+ // 300s of work sits under the (target - overhead) budget, so the margin must
+ // not be what pushes it over into a two-way split.
+ const union = {}
+ for (let i = 0; i < 10; i++) {
+ union[`products/mid_one/backend/test_${i}.py::test_${i}`] = 30
+ }
+
+ assert.ok(300 <= TARGET_WALL_SECONDS - PRODUCT_JOB_OVERHEAD_SECONDS)
+ assert.equal(productSplitShards({ work: 300, heavyCount: 0, lightWork: 300, maxLight: 30, testCount: 10 }), 1)
+
+ const matrix = buildMatrix(['mid-one'], union, true)
+
+ assert.equal(matrix.length, 1)
+ assert.equal(matrix[0].group, 'mid-one')
+})
+
+test("a split product's last shard absorbs a small product without leaking split flags", () => {
+ const union = {}
+ for (let i = 0; i < 11; i++) {
+ union[`products/big_one/backend/test_${i}.py::test_${i}`] = 30
+ }
+ union['products/small_one/backend/test_s.py::test_s'] = 40
+
+ assert.equal(productSplitShards({ work: 330, heavyCount: 0, lightWork: 330, maxLight: 30, testCount: 11 }), 2)
+
+ const matrix = buildMatrix(['big-one', 'small-one'], union, true)
+
+ // Two jobs, not three: small-one rides along in the lighter second shard.
+ assert.equal(matrix.length, 2)
+ const shared = matrix.find((entry) => entry.group.includes('small-one'))
+ assert.equal(shared.group, 'big-one (2/2), small-one')
+ assert.equal(shared.legs.length, 2)
+ assert.match(shared.legs[0].pytest_args, /--splits 2 --group 2/)
+ // The whole product runs in its own leg, so it never sees --splits/--group.
+ assert.equal(shared.legs[1].filters, '--filter=@posthog/products-small-one')
+ assert.equal(shared.legs[1].pytest_args, '')
+})
diff --git a/.github/scripts/turbo-discover.js b/.github/scripts/turbo-discover.js
index 82acf41bbabd..b548f33a5019 100644
--- a/.github/scripts/turbo-discover.js
+++ b/.github/scripts/turbo-discover.js
@@ -40,26 +40,32 @@ const { analyzeSchemaImpact, readBaseSchema } = require('./schema-impact')
const { loadContractSurfaces } = require('./trunk-impacted-targets')
// --- Product shard sizing (same Amdahl shape as Django below) ---
-// Each product is atomic for packing, but unlike Django the test pool isn't
-// fungible across products — bin-pack products into target-sized shards, and
-// multi-shard split any single product that overflows on its own.
+// The test pool is not fungible across products, so a product is the unit of
+// work: bin-pack products into target-sized jobs, and multi-shard split any
+// single product that overflows on its own. A job runs what it holds
+// sequentially, so its wall is the sum of its parts, not the max.
// One flat wall-clock target for every test shard, Django and products alike.
// Predictability is the point: a dev who kicks off CI knows what a shard costs
// without knowing which segment it is. Sizing solves wall = overhead + work/n
// for n, so the target is a promise about the PR lane (where the overheads below
// are fitted); master pays extra overhead (full migration replay) on top.
-// A full run's wall is discovery plus the slowest of its shards, and with many
-// shards packed to one target the slowest lands a few minutes above it, so a
-// 12-minute shard target puts a full PR run near 15 minutes end to end.
+// A full run's wall is the pre-shard preamble plus the slowest of its shards.
+// The preamble (discovery, matrix build, runner start) measures ~5.5 min, and
+// sizing bounds the slowest shard at the target rather than the average, so a
+// 12-minute shard target puts a full PR run near 18 minutes end to end.
const TARGET_WALL_SECONDS = 12 * 60
// Per-product cost within a runner: turbo dispatch, pytest collection, Django
// init. First product pays ~45s, subsequent ~15s; use 60s as a conservative
// average that also absorbs the amortized portion of runner startup.
const PRODUCT_PER_PRODUCT_OVERHEAD_SECONDS = 60
-// Headroom for run-to-run variance when deciding how much fits in a bucket. Was
-// 2x originally because pytest-split data was noisy under Django Core's shared
-// session; the outlier-based merge produces cleaner numbers now.
-const PRODUCT_SAFETY_FACTOR = 1.3
+// Headroom on a packed bucket, covering error in the recorded durations alone.
+// A bucket runs its products sequentially, so its wall is the sum of its parts
+// and it needs no allowance for an uneven split. That allowance belongs to the
+// split path, which derives its own in productSplitShards.
+const PRODUCT_BUCKET_SAFETY_FACTOR = 1.1
+// No headroom constant for a split product: the gap between the mean shard that
+// sizing solves for and the max shard that sets the wall is derived per product
+// in productSplitShards below.
// Fitted per-shard overhead for a split product job. Two measured parts, from
// run 32717208712: the job base (docker stack, deps, turbo dispatch) is
// mean(job wall - JUnit suite time), 247-413s across 12 bucket jobs (median
@@ -96,10 +102,11 @@ const PRODUCTS_RUNNING_TEMPORAL_IN_JOB = new Set([
'tasks',
'warehouse-sources',
])
-// Products that always get their own matrix entry instead of being packed with
-// others — isolates a flaky/hang-prone product so it can't cancel bucket-mates
-// at the job timeout. Trade-off: a dedicated runner.
-const DEDICATED_BUCKET_PRODUCTS = new Set(['batch-exports'])
+// Products that always get their own matrix entry instead of sharing one, so a
+// hang cannot cancel job-mates when the job timeout fires. The cost is a
+// dedicated runner, so a product belongs here only while its wall runs close
+// enough to the job timeout that a hang is a realistic outcome.
+const DEDICATED_BUCKET_PRODUCTS = new Set()
// --- Staleness detection for .test_durations ---
// When a product's test files on disk significantly outnumber what .test_durations
@@ -696,6 +703,43 @@ function getProductDuration(product, durations) {
return total
}
+// The longest single test in a product. pytest-split cuts between tests, never
+// inside one, so this is the irreducible grain of any split and it bounds how
+// far the worst chunk can run past the mean.
+// Budget of test work one product shard can hold, mirroring calculateShards.
+function productShardBudget() {
+ return Math.max(TARGET_WALL_SECONDS - PRODUCT_JOB_OVERHEAD_SECONDS, PRODUCT_JOB_OVERHEAD_SECONDS / 2, 1)
+}
+
+// The parts of a product's duration distribution that sizing needs. Two tests
+// longer than half a shard's budget can never share a shard, so those are counted
+// rather than summed; the rest are summed, with their own longest, because a
+// contiguous chunk of them runs at most one of them past the mean.
+function getProductShape(product, durations) {
+ const shape = { work: 0, maxTest: 0, heavyCount: 0, lightWork: 0, maxLight: 0, testCount: 0 }
+ if (!durations) {
+ return shape
+ }
+ const prefix = productPrefix(product)
+ const excluded = PRODUCTS_RUNNING_TEMPORAL_IN_JOB.has(product) ? [] : EXCLUDED_PATH_SEGMENTS
+ const heavyThreshold = productShardBudget() / 2
+ for (const [test, dur] of Object.entries(durations)) {
+ if (!test.startsWith(prefix) || excluded.some((seg) => test.includes(seg))) {
+ continue
+ }
+ shape.work += dur
+ shape.testCount += 1
+ shape.maxTest = Math.max(shape.maxTest, dur)
+ if (dur > heavyThreshold) {
+ shape.heavyCount += 1
+ } else {
+ shape.lightWork += dur
+ shape.maxLight = Math.max(shape.maxLight, dur)
+ }
+ }
+ return shape
+}
+
// One definition of a product's work estimate, shared by the split decision
// (buildMatrix) and the bucket cost (packProducts), so they cannot disagree.
//
@@ -707,39 +751,56 @@ function getProductDuration(product, durations) {
// under-sharding. `staleUnionWork` is non-null exactly when the guess replaced
// the recorded sum, so the caller can log it once.
function resolveProductSizing(product, durations, productsScaled = false) {
- const unionWork = getProductDuration(product, durations)
- if (productsScaled && unionWork > 0) {
- return { work: unionWork, staleUnionWork: null, staleness: null }
+ const shape = getProductShape(product, durations)
+ if (productsScaled && shape.work > 0) {
+ return { ...shape, staleUnionWork: null, staleness: null }
}
const staleness = checkProductStaleness(product, durations)
if (staleness.stale && staleness.fileCount > 0) {
const fallbackWork = staleness.fileCount * STALENESS_FALLBACK_SECONDS_PER_FILE
- if (fallbackWork > unionWork) {
- return { work: fallbackWork, staleUnionWork: unionWork, staleness }
+ if (fallbackWork > shape.work) {
+ // The tests the map does record are still measurements, and a heavy one
+ // holds a shard whatever the coverage. Keep those and treat only the
+ // guessed remainder as light, at one file's worth per test.
+ const recordedHeavyWork = shape.work - shape.lightWork
+ return {
+ work: fallbackWork,
+ maxTest: Math.max(shape.maxTest, STALENESS_FALLBACK_SECONDS_PER_FILE),
+ heavyCount: shape.heavyCount,
+ lightWork: Math.max(fallbackWork - recordedHeavyWork, 0),
+ maxLight: Math.max(shape.maxLight, STALENESS_FALLBACK_SECONDS_PER_FILE),
+ testCount: Math.max(shape.testCount, staleness.fileCount),
+ staleUnionWork: shape.work,
+ staleness,
+ }
}
}
- return { work: unionWork, staleUnionWork: null, staleness: null }
+ return { ...shape, staleUnionWork: null, staleness: null }
}
function productEffectiveCost(product, durations, productsScaled = false) {
const { work } = resolveProductSizing(product, durations, productsScaled)
- return work * PRODUCT_SAFETY_FACTOR + PRODUCT_PER_PRODUCT_OVERHEAD_SECONDS
+ return work * PRODUCT_BUCKET_SAFETY_FACTOR + PRODUCT_PER_PRODUCT_OVERHEAD_SECONDS
}
-// First-fit-decreasing bin packing into TARGET-sized shards. Sorts products by
+// First-fit-decreasing bin packing into TARGET-sized jobs. Sorts products by
// effective cost descending so the largest products land first and small ones
-// fill the gaps. Each bucket caps at the wall target minus the base overhead the
-// job pays once, so the effective costs only compete for the remaining budget.
-function packProducts(products, durations, productsScaled = false) {
+// fill the gaps. Each job caps at the wall target minus the base overhead it
+// pays once, so the effective costs only compete for the remaining budget.
+// `seedJobs` are jobs that already hold work — a split product's last shard —
+// and they sit first so their leftover budget is used before a new runner is
+// started. A seed carries its own base overhead, which is a large product's
+// session cost rather than the packed-bucket base.
+function packProducts(products, durations, productsScaled = false, seedJobs = []) {
const items = products
.map((product) => ({ product, cost: productEffectiveCost(product, durations, productsScaled) }))
.sort((a, b) => b.cost - a.cost)
- const buckets = []
+ const buckets = [...seedJobs]
for (const { product, cost } of items) {
let placed = false
for (const bucket of buckets) {
- if (bucket.cost + cost <= TARGET_WALL_SECONDS - PRODUCT_JOB_BASE_OVERHEAD_SECONDS) {
+ if (bucket.cost + cost <= TARGET_WALL_SECONDS - bucket.baseOverhead) {
bucket.products.push(product)
bucket.cost += cost
placed = true
@@ -747,7 +808,13 @@ function packProducts(products, durations, productsScaled = false) {
}
}
if (!placed) {
- buckets.push({ products: [product], cost })
+ buckets.push({
+ label: null,
+ legs: [],
+ products: [product],
+ cost,
+ baseOverhead: PRODUCT_JOB_BASE_OVERHEAD_SECONDS,
+ })
}
}
return buckets
@@ -833,6 +900,47 @@ function calculateShards(totalWorkSeconds, overheadSeconds, minShards = DJANGO_M
return Math.max(minShards, Math.min(DJANGO_MAX_SHARDS, shards))
}
+// Shards for one product. Sizing a split by work/n sizes the MEAN shard, but the
+// run's wall is the MAX shard, and pytest-split cuts between tests rather than
+// inside one, so size the worst chunk instead.
+//
+// Split the suite at half the budget. Two tests above that cannot share a shard
+// at all, so each takes one and they set a floor no packing goes below. What is
+// left is at most half a budget per test, so a contiguous chunk of it runs at
+// most one such test past its mean, giving lightWork/n + maxLight <= budget and
+// so n = ceil(lightWork / (budget - maxLight)). That denominator is at least
+// half the budget, so it cannot collapse.
+//
+// The cuts are contiguous, so a heavy test sitting between light ones divides
+// the light run rather than lifting out of it. H heavy tests leave at most H + 1
+// light runs, and each run rounds up on its own, so the light side can cost H
+// shards beyond its own bound. Charge that whenever any light work exists.
+//
+// That charge assumes a fragmentation the suite may not have, so cap the count
+// at the number of tests. Past it a shard is guaranteed to collect nothing
+// (pytest exit 5) and spends a runner without shortening the critical path.
+//
+// Reading the distribution rather than a fitted ratio ties the sizing to the
+// map: a suite of heavy tests gets the shards they force, an evenly grained one
+// gets none it does not need, and no constant carries a past map's error.
+//
+// A product whose whole suite fits one shard is not split, and the bound does
+// not apply to it -- an unsplit chunk is the work itself, with nothing on top.
+function productSplitShards(shape) {
+ const budget = productShardBudget()
+ const { work = 0, heavyCount = 0, lightWork = 0, maxLight = 0, testCount = Infinity } = shape ?? {}
+ if (work <= budget) {
+ return 1
+ }
+ const lightShards = lightWork > 0 ? Math.ceil(lightWork / (budget - maxLight)) : 0
+ const fragmentation = lightWork > 0 ? heavyCount : 0
+ const wanted = Math.min(heavyCount + lightShards + fragmentation, testCount)
+ // The two-shard floor cannot outrank the test count: a product holding one
+ // test that overruns the budget still gets one job, because the second would
+ // collect nothing and splitting cannot shorten the first.
+ return Math.max(Math.min(2, testCount), Math.min(DJANGO_MAX_SHARDS, wanted))
+}
+
// Selector segment key -> Django matrix segment name.
const MATRIX_NAME_BY_SEGMENT = { core: 'Core', poe: 'CorePOE', temporal: 'Temporal' }
@@ -986,17 +1094,34 @@ function buildDjangoShards(durations, ranNodeIds = {}) {
return result
}
+// A workflow edit reaches an open PR before this script does, so an entry a
+// single turbo invocation can express keeps the pre-legs {filters, pytest_args}
+// keys beside its leg. An entry with several legs has no such expression and
+// carries legs alone, by which point the workflow reading it is the new one.
+function matrixEntry(group, legs) {
+ const entry = { group, legs }
+ if (legs.length === 1) {
+ entry.filters = legs[0].filters
+ entry.pytest_args = legs[0].pytest_args
+ }
+ return entry
+}
+
function buildMatrix(products, durations, productsScaled = false) {
const matrix = []
const packable = []
+ const fillableJobs = []
// Split a product across multiple shards with the same rule Django uses:
- // enough shards that each lands at the shared wall target. The safety
- // factor applies here as it does to packing: the products that split are
- // the fixture-heavy suites whose recorded durations undercount the most,
- // and a split sized on the bare sum lands its shards well past the target.
+ // enough shards that each lands at the shared wall target. Unlike packing,
+ // the split carries no safety factor -- productSplitShards derives its own
+ // headroom from the product's longest test instead. That leaves it trusting
+ // the recorded sum, which holds only while the map carries
+ // PRODUCTS_SCALED_MARKER: call-only durations undercount a fixture-heavy
+ // suite several-fold, and sizing an unscaled sum under-shards it.
for (const product of products) {
- const { work, staleUnionWork, staleness } = resolveProductSizing(product, durations, productsScaled)
+ const sizing = resolveProductSizing(product, durations, productsScaled)
+ const { work, maxTest, staleUnionWork, staleness } = sizing
if (staleUnionWork !== null) {
console.error(
` ${product}: .test_durations stale, ${staleness.coveredCount}/${staleness.fileCount} test files covered ` +
@@ -1008,7 +1133,7 @@ function buildMatrix(products, durations, productsScaled = false) {
)
}
- const shards = calculateShards(work * PRODUCT_SAFETY_FACTOR, PRODUCT_JOB_OVERHEAD_SECONDS, 1)
+ const shards = productSplitShards(sizing)
if (shards > 1) {
console.error(` ${product}: ${(work / 60).toFixed(1)} min work → split across ${shards} shards`)
const filters = `--filter=@posthog/products-${product}`
@@ -1017,34 +1142,47 @@ function buildMatrix(products, durations, productsScaled = false) {
// optimally. The greedy rule in duration_based_chunks lets every shard
// overrun the per-shard average, which on skewed suites starves trailing
// shards down to zero tests (pytest exit 5, "no tests collected").
+ const shardCost = work / shards + maxTest
for (let i = 1; i <= shards; i++) {
- matrix.push({
- group: `${product} (${i}/${shards})`,
+ const leg = {
filters,
pytest_args: `-- --splits ${shards} --group ${i} --splitting-algorithm optimal_chunks`,
- })
+ }
+ // work/shards + maxTest bounds every shard, whichever one
+ // optimal_chunks leaves lightest, so one shard can be offered to the
+ // packer without knowing which. Do not tighten this to work/shards:
+ // the bound is what keeps a filled shard inside the job budget.
+ if (i === shards && !DEDICATED_BUCKET_PRODUCTS.has(product)) {
+ fillableJobs.push({
+ label: `${product} (${i}/${shards})`,
+ legs: [leg],
+ products: [],
+ cost: shardCost,
+ baseOverhead: PRODUCT_JOB_OVERHEAD_SECONDS,
+ })
+ } else {
+ matrix.push(matrixEntry(`${product} (${i}/${shards})`, [leg]))
+ }
}
} else if (DEDICATED_BUCKET_PRODUCTS.has(product)) {
- console.error(` ${product}: ${(work / 60).toFixed(1)} min work → dedicated bucket (never packed)`)
- matrix.push({
- group: product,
- filters: `--filter=@posthog/products-${product}`,
- pytest_args: '',
- })
+ console.error(` ${product}: ${(work / 60).toFixed(1)} min work → dedicated job (never shared)`)
+ matrix.push(matrixEntry(product, [{ filters: `--filter=@posthog/products-${product}`, pytest_args: '' }]))
} else {
packable.push(product)
}
}
- for (const bucket of packProducts(packable, durations, productsScaled)) {
- console.error(
- ` bucket (${(bucket.cost / 60).toFixed(1)} min effective): ${bucket.products.join(', ')}`
- )
- matrix.push({
- group: bucket.products.join(', '),
- filters: bucket.products.map((p) => `--filter=@posthog/products-${p}`).join(' '),
- pytest_args: '',
- })
+ for (const bucket of packProducts(packable, durations, productsScaled, fillableJobs)) {
+ const group = [bucket.label, ...bucket.products].filter(Boolean).join(', ')
+ console.error(` job (${(bucket.cost / 60).toFixed(1)} min effective): ${group}`)
+ const legs = [...bucket.legs]
+ if (bucket.products.length > 0) {
+ legs.push({
+ filters: bucket.products.map((p) => `--filter=@posthog/products-${p}`).join(' '),
+ pytest_args: '',
+ })
+ }
+ matrix.push(matrixEntry(group, legs))
}
return matrix
@@ -1062,7 +1200,9 @@ module.exports = {
resolveProductSizing,
buildMatrix,
PRODUCT_JOB_OVERHEAD_SECONDS,
- PRODUCT_SAFETY_FACTOR,
+ PRODUCT_BUCKET_SAFETY_FACTOR,
+ productSplitShards,
+ getProductShape,
PRODUCTS_SCALED_MARKER,
TARGET_WALL_SECONDS,
DJANGO_OVERHEAD_SECONDS_BY_SEGMENT,
diff --git a/.github/workflows/ci-backend-update-test-timing.yml b/.github/workflows/ci-backend-update-test-timing.yml
index fbde480bc553..ca341a4899aa 100644
--- a/.github/workflows/ci-backend-update-test-timing.yml
+++ b/.github/workflows/ci-backend-update-test-timing.yml
@@ -155,7 +155,8 @@ jobs:
# The merge below takes product entries only from the products file
# (--replace-prefix), so a Products slice that fails the drift check
# needs a stand-in or every product loses its durations. The last
- # cached map is that stand-in. A miss leaves the checked-out file.
+ # cached map is that stand-in. .test_durations is untracked, so a miss
+ # leaves no file and the fallback below publishes no product entries.
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
continue-on-error: true
with:
@@ -166,11 +167,17 @@ jobs:
posthog-test-durations-
- name: Set the previous map aside
- # The restore lands on the tracked file; keep the copy elsewhere and put
- # the committed file back so the change check below diffs a clean tree.
+ # Keep the restored map where the drift fallback below expects it, and
+ # take it out of the tree so the fresh map is written rather than merged
+ # onto the old one. .test_durations is not tracked (it left the repo when
+ # the timing workflow started publishing it as an artifact), so there is
+ # nothing to restore it from and a cache miss leaves no file at all.
run: |
- cp .test_durations /tmp/previous_durations
- git checkout -- .test_durations
+ if [ -f .test_durations ]; then
+ mv .test_durations /tmp/previous_durations
+ else
+ echo "::warning::No cached map restored; a drifting Products slice has no stand-in"
+ fi
- name: Download timing artifacts
env:
@@ -265,12 +272,21 @@ jobs:
set -e
if [ "$products_status" -ne 0 ]; then
echo "::warning::Products durations refresh exited $products_status; keeping the previous products slice. Drift or an incomplete JUnit set are the expected causes, see the log above."
- # Only entries the fresh slices do not carry: the outlier merge would
- # otherwise let a stale value here beat a fresh one for the same test.
- jq -s 'add' $(ls /tmp/core_durations /tmp/temporal_durations /tmp/dagster_durations 2>/dev/null) > /tmp/fresh_durations
- jq --slurpfile fresh /tmp/fresh_durations \
- 'with_entries(select((.key | startswith("products/")) and ($fresh[0][.key] == null)))' \
- /tmp/previous_durations > /tmp/products_durations
+ if [ -f /tmp/previous_durations ]; then
+ # Only entries the fresh slices do not carry: the outlier merge would
+ # otherwise let a stale value here beat a fresh one for the same test.
+ jq -s 'add' $(ls /tmp/core_durations /tmp/temporal_durations /tmp/dagster_durations 2>/dev/null) > /tmp/fresh_durations
+ jq --slurpfile fresh /tmp/fresh_durations \
+ 'with_entries(select((.key | startswith("products/")) and ($fresh[0][.key] == null)))' \
+ /tmp/previous_durations > /tmp/products_durations
+ else
+ # No cached slice to stand in and no fresh one. A map with no product
+ # entries would still become the newest cache and the latest successful
+ # artifact, so every consumer would fall back to the file-count estimate
+ # and under-shard the fixture-heavy products. Leave the older map newest.
+ echo "::error::Products slice failed with no previous map to stand in; refusing to publish a map without product entries"
+ exit 1
+ fi
fi
# Process Dagster segment if available
diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
index 559638db70f4..38617b616a6a 100644
--- a/.github/workflows/ci-backend.yml
+++ b/.github/workflows/ci-backend.yml
@@ -965,8 +965,15 @@ jobs:
continue-on-error: true
# --force: discover already decided this product needs testing, skip turbo cache
# --log-order=stream: stream pytest output live instead of buffering until completion
- # pytest_args: optional pytest-split flags for sharded products (e.g. "-- --splits 3 --group 1")
+ # legs: one turbo invocation per entry, each with its own pytest-split flags
+ # (e.g. "-- --splits 3 --group 1"). A job holds several legs when a split
+ # product's last shard had budget left over for whole small products, and
+ # pytest-split flags must not leak from that shard onto its job-mates.
env:
+ PRODUCT_LEGS: ${{ toJSON(matrix.legs) }}
+ # Read only when an unrebased branch planned the pre-legs shape.
+ LEGACY_FILTERS: ${{ matrix.filters }}
+ LEGACY_PYTEST_ARGS: ${{ matrix.pytest_args }}
CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
# products/tasks/backend/temporal moved here from the Django Temporal
# segment, and its conftest and workflows talk to Modal. Injected only
@@ -993,17 +1000,42 @@ jobs:
# bin-packed buckets (each product writes its own file) and split products (unioned later).
# sysmon: Python 3.12+'s low-overhead coverage backend, ~a few % vs ~20% for the C tracer.
COVERAGE_CORE: sysmon
+ shell: bash
run: |
set +e
- pnpm turbo run backend:test ${{ matrix.filters }} --concurrency=1 --output-logs=full --force --log-order=stream ${{ matrix.pytest_args }}
- exit_code=$?
- set -e
- if [ $exit_code -eq 5 ]; then
- echo "No tests collected for this shard, this is expected when splitting tests"
- exit 0
- else
- exit $exit_code
+ overall=0
+ # A workflow edit reaches an open PR before the script edit does, so a
+ # branch that has not rebased still plans the flat {filters, pytest_args}
+ # shape. Normalize that into one leg rather than requiring the new key.
+ if [ -z "$PRODUCT_LEGS" ] || [ "$PRODUCT_LEGS" = "null" ]; then
+ PRODUCT_LEGS=$(jq -cn --arg f "$LEGACY_FILTERS" --arg a "$LEGACY_PYTEST_ARGS" \
+ '[{filters: $f, pytest_args: $a}]')
+ echo "Matrix entry carries no legs; running the legacy single invocation"
fi
+ leg_count=$(jq 'length' <<< "$PRODUCT_LEGS")
+ # Neither shape present means the job would pass having run no tests, and
+ # the rollup gate only reads this step's outcome. Fail loudly instead.
+ if [ -z "$leg_count" ] || [ "$leg_count" -lt 1 ]; then
+ echo "::error::No test legs in this matrix entry; refusing to pass without running tests"
+ exit 1
+ fi
+ for ((leg_index = 0; leg_index < leg_count; leg_index++)); do
+ leg_filters=$(jq -r ".[$leg_index].filters" <<< "$PRODUCT_LEGS")
+ leg_args=$(jq -r ".[$leg_index].pytest_args" <<< "$PRODUCT_LEGS")
+ echo "::group::turbo backend:test $leg_filters $leg_args"
+ # Word splitting is intended here: both variables carry several flags.
+ # shellcheck disable=SC2086
+ pnpm turbo run backend:test $leg_filters --concurrency=1 --output-logs=full --force --log-order=stream $leg_args
+ exit_code=$?
+ echo "::endgroup::"
+ if [ $exit_code -eq 5 ]; then
+ echo "No tests collected for this leg, this is expected when splitting tests"
+ elif [ $exit_code -ne 0 ]; then
+ overall=$exit_code
+ fi
+ done
+ set -e
+ exit $overall
# Excluded reports bypass Trunk quarantine, so this step owns their verdict.
- name: Prepare product JUnit for Trunk
diff --git a/docs/internal/ci-things-already-tried.md b/docs/internal/ci-things-already-tried.md
new file mode 100644
index 000000000000..2af104cc4372
--- /dev/null
+++ b/docs/internal/ci-things-already-tried.md
@@ -0,0 +1,498 @@
+# CI: things already tried
+
+This file answers one question. **You have an idea for CI or the dev environment. Did someone try it before?**
+
+Most of the ideas here are good ideas. People tried them because they sounded correct.
+This file records the part that costs the most to find again. It records what happened when someone built the idea, and why the result did not agree with the proposal.
+
+## How to use this file
+
+Search before you build.
+
+```bash
+rg -i "xdist|parallel" docs/internal/ci-things-already-tried.md
+```
+
+Each entry has the title of the **proposal**. The title uses the words that a person uses to propose the idea.
+The title does not use the symptom that appeared later.
+Each entry ends with _Also asked as_. This line gives other words for the same idea, so that a different search finds the entry.
+
+**A verdict is not a prohibition.** Each entry gives the date and the specific reason for the result.
+Read the reason. Then examine if the reason is still correct. Runner sizes, prices, and tools change.
+If the reason is no longer correct, write this in the PR and try the idea again.
+
+## Verdicts
+
+| Verdict | Meaning |
+| ------------ | --------------------------------------------------------------- |
+| `rejected` | Someone built the idea and measured it. The result was too bad. |
+| `reverted` | The change went to master. Then someone removed it. |
+| `superseded` | The problem was real. A different solution replaced this one. |
+| `abandoned` | Someone started the work and stopped. There is no verdict. |
+| `open` | The idea is good. The work is incomplete. You can continue it. |
+
+## Add an entry
+
+Add an entry when you close a PR and do not merge it. Add an entry when you revert a change.
+Give the entry the title of the idea. Then give the verdict, the date, and the measurement.
+An entry of five lines has more value than a design document that nobody opens.
+
+## Remove an entry
+
+Age alone is not a reason to remove an entry. The pytest-xdist entry is the oldest here, and people still propose that idea.
+
+Remove an entry when one of these is true:
+
+The system that it describes is gone. A person cannot propose the idea any more, so the verdict guides nobody.
+
+The idea shipped later. The entry is history, not prior art. Keep it only when the first failure is still a trap.
+
+The entry gives a general lesson and no specific trap. "Measure before you optimize" does not need an entry.
+
+Give the reason when you remove an entry. Do not remove an entry because it looks old.
+
+---
+
+## Test parallelism and sharding
+
+### Run pytest-xdist inside the backend CI shards
+
+**Verdict: rejected** · Oct 2025 · [#38927](https://github.com/PostHog/posthog/pull/38927)
+
+The test used 53 shards and `-n 4`.
+Wall time decreased from approximately 15 minutes to approximately 9 minutes. The PR reports the difference as statistically strong.
+CPU cost increased from 1,572 to 3,908 core-minutes. This is a factor of approximately 2.5.
+
+The speed increase is real. The cost is the problem.
+A factor of 2.5 in compute is too much for 6 minutes of wall time.
+
+`pytest-xdist` is still a development dependency, and it operates correctly on a local machine. CI does not use it in the shards.
+
+_Also asked as:_ parallelize tests within a shard, `-n auto`, use the idle cores on the runner, why is each shard single-process
+
+### Change the `django_db_setup` fixture from package scope to session scope
+
+**Verdict: rejected** · Apr 2026 · [#57030](https://github.com/PostHog/posthog/pull/57030)
+
+Package scope builds the test database one time for each package directory. Session scope builds it one time for the whole run, which looks strictly faster.
+
+The PR did not merge. `posthog/conftest.py` still declares `@pytest.fixture(scope="package")`.
+Read [#57227](https://github.com/PostHog/posthog/pull/57227) with this one. It makes the cost of `django_db_setup` visible in the pytest output. Today that cost hides in the setup phase of the first test that pytest collects, and makes that test look slow for no reason.
+
+Measure the setup cost first. Then you know what a scope change can win.
+
+_Also asked as:_ session-scoped database fixture, build the test database once, why is the first test so slow
+
+### Shard the Playwright E2E suite
+
+**Verdict: reverted** · Feb 2026 · [#46774](https://github.com/PostHog/posthog/pull/46774), reverted by [#46853](https://github.com/PostHog/posthog/pull/46853)
+
+The change added four shards. The purpose was a smaller retry set for unreliable tests.
+
+The setup cost of each shard is the item that the proposal did not include. Each shard needs approximately 7.5 minutes of setup. The migrations alone need 3 minutes.
+
+All 110 tests complete in approximately 4 minutes with 6 workers.
+Thus four shards used approximately 22 more minutes of CPU in each run, and decreased wall time by approximately 3 minutes.
+
+A shard retry also repeats the 7.5 minutes of setup. Thus a retry of one shard is not much faster than a retry of all the tests.
+
+The workflow contains a comment about this decision. Read the comment near `runs-on` in `.github/workflows/ci-e2e-playwright.yml`.
+
+_Also asked as:_ split the E2E tests across runners, parallelize Playwright, reduce flaky retry scope by sharding
+
+### Use Bazel to scope product tests
+
+**Verdict: abandoned** · Dec 2025 to Mar 2026 · [#43397](https://github.com/PostHog/posthog/pull/43397)
+
+Each product could select Bazel targets. Then a change to one product could skip the legacy pytest jobs.
+
+The branch became inactive, and the stale bot closed it. Nobody measured the result. Thus this entry is not evidence against Bazel.
+
+A different solution achieved the same goal. The product tests moved to Turborepo in [#46971](https://github.com/PostHog/posthog/pull/46971). File-level backend selection came later.
+
+_Also asked as:_ Bazel, build graph for test selection, only run tests for the product I changed
+
+## Test selection
+
+### Exclude `products/**/backend/**` from the backend paths filter
+
+**Verdict: reverted** · Mar 2026 · [#50137](https://github.com/PostHog/posthog/pull/50137), reverted by [#50181](https://github.com/PostHog/posthog/pull/50181)
+
+The filter gave the decision to `contract-check`. `contract-check` decides if the Django tests are necessary for a change in a product.
+
+This assumes that the products are isolated. Most products were not isolated.
+Core code imports product views, serializers, and models directly. It imports them through `posthog/api/__init__.py`, `posthog/tasks/`, and the migrations.
+`contract-check` examines only the facade files. Thus it cannot see these imports.
+
+The rule is general. A path filter that skips tests makes a statement about the import graph. Examine the import graph before you make the statement.
+
+_Also asked as:_ narrow the backend paths filter, skip Django tests for product-only changes, trust contract-check
+
+### Certify a facade with its `__all__`
+
+**Verdict: superseded** · Jul 2026 · [#71127](https://github.com/PostHog/posthog/pull/71127), replaced by [#71486](https://github.com/PostHog/posthog/pull/71486)
+
+The detector read `__all__` to prove that a facade does not re-export internal names.
+
+Two problems stopped it. Each review round found a new gap in the detector.
+Also, most facade modules do not declare `__all__`. Thus the detector could certify only the part of the surface that the module declares.
+
+The replacement puts the rule in `products/architecture.md`. It also makes the `contract-check` inputs narrow or absent, instead of a list of file globs.
+
+_Also asked as:_ detect facade leaks, check `__all__`, verify a product is really isolated
+
+### Select tests from pytest-testmon coverage data
+
+**Verdict: superseded** · Apr 2026 · [#56370](https://github.com/PostHog/posthog/pull/56370)
+
+This PR collected the data. It did not connect the selection to CI.
+The merge of 52 shard artifacts gave a map of 28,322 tests over 3,691 production files. The map has approximately 1.3 million entries.
+
+The selectivity was high. One changed file caused a median of 45 tests. This is a skip rate of 99.8%.
+
+Two results have more importance than these numbers.
+First, there were no stale tests with high confidence. Thus a cleanup from this data had nothing to delete.
+Second, 1,020 tests appeared to touch no production code. Almost all of these results are false. They come from code with many mocks, from property-based tests, and from tests of migration rules. Testmon cannot trace these paths.
+
+Backend test selection came later from a different mechanism. It uses the Snob import graph. Read the next entry.
+
+_Also asked as:_ coverage-based test selection, testmon, find stale tests from coverage, only run affected tests
+
+### Snob is in CI, but CI does not use it
+
+**Verdict: CI does use it.** The scope was narrow on purpose.
+
+`tools/snob_backend_test_selection_shadow.py` selects the Django test subset for a PR. It combines the Snob import graph with Django-aware heuristics.
+
+Two things make this look inactive:
+
+`pytest-snob` is an inline PEP 723 dependency of that script. It is not in `pyproject.toml`. Thus a search of the dependency file finds nothing.
+The selection was also active for draft PRs only for some time. The team wanted a stable merge queue first.
+
+[#85530](https://github.com/PostHog/posthog/pull/85530) then extended the selection to PRs that are ready for review. [#88265](https://github.com/PostHog/posthog/pull/88265) put the Django selection and the product selection in one job.
+Read the comment at the top of `.github/workflows/ci-backend.yml` for the current rules.
+
+_Also asked as:_ snob, is test selection on, why does CI run all the tests, do we select tests on PRs
+
+### Disable the pytest `unraisableexception` and `threadexception` plugins
+
+**Verdict: open, and approved** · Jul 2026 · [#70886](https://github.com/PostHog/posthog/pull/70886)
+
+Each pytest session runs several full-heap `gc.collect()` passes at cleanup.
+These plugins run the passes only to report `__del__` exceptions and thread exceptions as warnings.
+`addopts` already sets `-p no:warnings`. Thus these warnings cannot become failures, and the passes give no value.
+
+A fixed benchmark of 320 tests decreased from 24.7 seconds to 21.8 seconds.
+
+A reviewer approved the PR. The branch then became inactive, and the stale bot closed it.
+You can open this PR again without changes.
+
+Read this entry before you try a different solution for the pytest cleanup cost.
+[#88759](https://github.com/PostHog/posthog/pull/88759) tried a different solution. It deleted the `gc.unfreeze()` in `pytest_unconfigure`.
+That call is necessary. [#62707](https://github.com/PostHog/posthog/pull/62707) added it after the Temporal shards stopped with a segmentation fault and exit code 139. CI made the same crash again on #88759.
+Frozen objects do not get the final cyclic collections of `Py_FinalizeEx`. Thus their finalizers run late in the teardown, after Python removes the extension modules.
+
+_Also asked as:_ pytest teardown is slow, reduce gc.collect at session end, speed up pytest cleanup, why does the shard hang after the tests pass
+
+## Docker and image builds
+
+### Apply BuildKit cache mounts to the Dockerfile
+
+**Verdict: rejected** · Oct 2025 · [#39700](https://github.com/PostHog/posthog/pull/39700)
+
+The change added cache mounts for apt, pip, uv, node, and Playwright. It followed the documented guidance from Depot.
+
+A measurement against master with a warm cache showed a slower build.
+A backend change increased from 52.7 to 57.5 seconds. This is approximately 9% slower. A frontend change increased from 55.5 to 62.2 seconds. This is approximately 12% slower.
+
+The workload is the reason. A cache mount adds 5 to 7 seconds of overhead, even when the cache has the data. A cache mount gives a benefit only when the dependencies change.
+Approximately 95% of PRs change code and do not change dependencies. Thus the change made the frequent case slower to make the rare case faster.
+
+A narrow version came later and stayed. [#42124](https://github.com/PostHog/posthog/pull/42124) added a uv cache mount, and the Dockerfile also has pnpm and npm mounts today.
+That narrow version then caused its own failure. The uv cache kept wheels that were compiled against a different `libxmlsec1` version, and the build failed with a version mismatch.
+[#43066](https://github.com/PostHog/posthog/pull/43066) proposed to remove the mount again. [#43091](https://github.com/PostHog/posthog/pull/43091) gave the better fix: it puts the `libxmlsec1` version in the cache ID, so a change of the system library invalidates the cache.
+Read the `id=uv-libxmlsec1...` mount in the Dockerfile.
+
+The rule: add a cache mount for one expensive step that you measured. Do not add cache mounts everywhere. Put the version of any system library that the cached artifacts compile against in the cache ID.
+
+_Also asked as:_ `--mount=type=cache`, speed up Docker builds, follow Depot cache best practices, xmlsec version mismatch in the image build
+
+### Move the source COPY to the end of the Dockerfile
+
+**Verdict: rejected** · Oct 2025 · [#39695](https://github.com/PostHog/posthog/pull/39695)
+
+The source COPY was already at the end. The layer order in the Dockerfile was already correct. Thus the change did nothing.
+
+Remember this type of proposal. Examine the current state before you optimize it.
+
+_Also asked as:_ improve Docker layer caching, reorder Dockerfile layers
+
+### Examine the Docker Hub credentials when CI reports a pull rate limit
+
+**Verdict: the cause was different** · Aug 2026 · [#81963](https://github.com/PostHog/posthog/pull/81963)
+
+CI failed with this message: `toomanyrequests: You have reached your unauthenticated pull rate limit`.
+At the maximum, this failure occurred in approximately 45% of the backend CI jobs. The usual rate is zero.
+
+The message indicates an authentication problem. This is why the diagnosis took a long time.
+`docker login` was successful during all of this period, and the message `Login Succeeded` was correct.
+
+The true cause was a lapse in the Docker Hub subscription. A lapsed plan removes the entitlement. Docker then issues anonymous tokens for the pulls, but it continues to accept the login.
+
+If you see this message again, examine the subscription status before you change the secrets.
+
+_Also asked as:_ Docker Hub rate limit in CI, unauthenticated pull limit, DOCKERHUB secret is wrong
+
+## CI orchestration
+
+### Move CI from the Depot runners to Blacksmith
+
+**Verdict: rejected** · Apr 2026 to May 2026 · [#54559](https://github.com/PostHog/posthog/pull/54559), removed by [#57991](https://github.com/PostHog/posthog/pull/57991)
+
+The trial did not do a direct exchange. It ran a Blacksmith shadow of most compute jobs on the same commit, behind the `BLACKSMITH_SHADOW_ENABLED` variable. Each shadow used `continue-on-error`, and no shadow was a required check.
+
+The team kept Depot. [#57991](https://github.com/PostHog/posthog/pull/57991) removed the shadow workflow and the matrix branches.
+The measurements are not in the PRs, so this entry cannot show them.
+It kept `.github/scripts/compare-ci-runners.py` and marked the file as legacy. That script produced the numbers of the trial.
+
+If you propose this again, equalize the caches of the two providers first. A runner that keeps a warm cache between jobs measures the cache, not the compute.
+Run the trial for several days. A short window cannot separate the jobs whose times are close.
+
+_Also asked as:_ change CI provider, Blacksmith, cheaper runners, are the Depot runners slow
+
+### Use sparse-checkout on the large CI workflows
+
+**Verdict: abandoned** · Oct 2025 · [#39239](https://github.com/PostHog/posthog/pull/39239)
+
+The description covers the backend, frontend, and Rust workflows. The diff changes only `ci-backend.yml` and `ci-rust.yml`.
+
+The PR did not merge, and no person reviewed it. The stale bot closed it. Thus there is no recorded reason for the result.
+
+Sparse-checkout is correct for small jobs, and `ci-storybook.yml`, `ci-security.yaml`, and `pr-resolve-outdated-bot-comments.yml` use it today.
+`ci-rust.yml` also uses it on the build and test jobs, but that came before this PR.
+The large Python and frontend test jobs do not use it. Their checkout is complete.
+
+If you propose this again, name the jobs and prove that each one reads only the included paths. A test job can read more of the tree than an exclusion list expects.
+
+_Also asked as:_ sparse-checkout, partial clone, do not check out the whole repo, speed up the checkout step
+
+### Jest reports the Rust snapshots as obsolete
+
+**Verdict: superseded** · Jan 2026 · [#46008](https://github.com/PostHog/posthog/pull/46008)
+
+Jest found the `.snap` files under `rust/cymbal/tests/snapshots/` during the Storybook visual regression job. It marked them as obsolete, and all 19 jobs failed.
+
+The PR records the attempts that did not work. `modulePathIgnorePatterns` changes only the module resolution. It does not change which snapshot files Jest finds.
+The PR proposed `haste.blockList`. The repository does not use that option today, so a different change solved this.
+
+Keep the record: the snapshot scan and the module resolution use different configuration.
+
+_Also asked as:_ obsolete snapshots in CI, Jest finds rust snapshots, modulePathIgnorePatterns
+
+### Skip Storybook and E2E for snapshot-only commits from the bot
+
+**Verdict: reverted** · Mar 2026 · [#49997](https://github.com/PostHog/posthog/pull/49997), reverted by [#51212](https://github.com/PostHog/posthog/pull/51212)
+
+The change went to master. One week later, a revert removed it.
+
+_Also asked as:_ skip CI for snapshot commits, ignore bot commits in CI, don't rerun visual tests for the snapshot bot
+
+### Force-cancel the backend CI run when pytest does not stop
+
+**Verdict: reverted** · Apr 2026 · [#54261](https://github.com/PostHog/posthog/pull/54261), reverted by [#54685](https://github.com/PostHog/posthog/pull/54685)
+
+The change added a watchdog job. The job cancels a run when pytest does not exit.
+The change stayed in master for one day.
+
+_Also asked as:_ cancel watchdog, kill hung CI jobs, pytest ignores SIGTERM
+
+### Add a CI step that asks a person to take a bot PR
+
+**Verdict: rejected** · Jun 2026 · [#62111](https://github.com/PostHog/posthog/pull/62111)
+
+CI cannot assign a PR from a bot automatically, because the bot account is not a member of the team.
+The CI step asked a person to take the PR.
+
+A different solution replaced it. A CI step must guess which person controls a bot PR.
+The agent that opens the PR already knows this person. Thus the instruction moved to the PR template.
+
+_Also asked as:_ auto-assign bot PRs, find the human behind an agent PR, nudge for ownership
+
+## Dev environment
+
+### Share the dev environment and the Docker containers across worktrees
+
+**Verdict: three attempts, none merged** · Oct 2025 to Apr 2026 · [#40634](https://github.com/PostHog/posthog/pull/40634), [#45984](https://github.com/PostHog/posthog/pull/45984), [#51100](https://github.com/PostHog/posthog/pull/51100)
+
+Each attempt used a different mechanism.
+[#40634](https://github.com/PostHog/posthog/pull/40634) changed the compose setup so a worktree uses the containers of the main checkout.
+[#45984](https://github.com/PostHog/posthog/pull/45984) set `COMPOSE_PROJECT_NAME` in the flox variables. Docker Compose uses the directory name when this variable is absent, so each worktree makes its own containers.
+[#51100](https://github.com/PostHog/posthog/pull/51100) shared the flox environment, the Python virtual environment, and `node_modules`. It reports approximately 5 GB of disk for each worktree.
+
+None of the three merged. `bin/wait-for-docker` gives the compose project the default name `posthog` today, which gives the shared containers that #45984 wanted.
+
+Read [#40634](https://github.com/PostHog/posthog/pull/40634) first if you propose this again. It asks the question that stopped all three: does any person need separate databases for each worktree?
+
+_Also asked as:_ worktrees start their own containers, share node_modules between worktrees, worktree disk usage, COMPOSE_PROJECT_NAME
+
+### Run a dmypy daemon for fast local type checks
+
+**Verdict: rejected** · Oct 2025 · [#39319](https://github.com/PostHog/posthog/pull/39319)
+
+A pre-commit hook used the daemon if the daemon was already active. A warm daemon gives a check of approximately 0.6 to 1.7 seconds.
+
+The warm-up time did not decrease. Thus the first check still has the full cost.
+A start of the daemon from mprocs was also tested. The commits then stopped and waited, because the daemon was still warm.
+
+_Also asked as:_ dmypy, speed up mypy locally, type-check on commit, mypy daemon
+
+### Run `uv sync` at each flox re-activation
+
+**Verdict: rejected** · Feb 2026 · [#49183](https://github.com/PostHog/posthog/pull/49183)
+
+The shell profiles synchronize the dependencies at each start of a shell. This needs approximately 660 ms when the dependencies are current.
+
+The reason for the rejection is the frequency. The profiles also run in subshells. Thus the cost occurs much more frequently than the problem.
+
+_Also asked as:_ auto-sync deps, keep the venv current automatically, uv sync in the shell profile
+
+### Upgrade Python to a version that the flox uv cannot install
+
+**Verdict: reverted** · Oct 2025 · [#40286](https://github.com/PostHog/posthog/pull/40286), reverted by [#40290](https://github.com/PostHog/posthog/pull/40290)
+
+Python 3.12.12 needs uv 0.9.2 or later. Flox pinned uv 0.8.23, and that version can get Python 3.12.10 at the maximum.
+Thus a person without Python 3.12.12 on the local machine could not build the environment.
+
+Remember the constraint, not the versions. The repository now uses Python 3.13.13, and this pin is obsolete.
+Before you increase the Python version, examine which versions the pinned flox uv can get.
+
+_Also asked as:_ bump Python, upgrade the interpreter, why is Python pinned to an exact version
+
+## Django performance
+
+[docs/internal/django-startup-time.md](django-startup-time.md) is the deep source for this area. It has a Traps section that records the failure modes of each mechanism.
+The entries below give the proposals that people repeat.
+
+### Squash the Django migration history
+
+**Verdict: rejected** · Feb 2026 to Mar 2026 · [#48267](https://github.com/PostHog/posthog/pull/48267)
+
+The PR added a squash planner, a policy for opaque operations, and 65 squashed migrations across the historical range. A zero-to-head migration on a fresh database was successful, and the schema comparison found no structural difference.
+
+The problem is the value. The PR reports that the effect on the timing was small and noisy. The work to resolve each blocker is large, and the reviews are difficult.
+
+[#60518](https://github.com/PostHog/posthog/pull/60518) tried a second angle three months later. It took the final project state at a cutoff date and rebuilt it as one set of `CreateModel` operations. The PR says that per-app squashing "only nibbles at it because the dep graph is cross-app". That PR also did not merge.
+
+The migration replay in CI is a real cost. Two different squash designs did not decrease it enough. A different change must decrease it.
+
+_Also asked as:_ squash the migrations, compress the migration history, why are there so many migrations, speed up the migration replay, nextgensquash
+
+### Build the generated pydantic schema lazily with `defer_build`
+
+**Verdict: reverted** · [docs/internal/django-startup-time.md](django-startup-time.md)
+
+This removed approximately 400 ms of core-schema construction from each `django.setup()`. The round-trip tests were all successful.
+
+Two problems stopped it. First, the deferred builds move to the first `/query` of each web worker after a deploy. A warm-up loop for those builds measured approximately 2.5 times more expensive than the eager construction.
+Second, the query runners construct the response models directly. This does no validation, so it does not start the lazy build. `model_dump()` then sends a mock serializer into pydantic-core and raises `TypeError: 'MockValSer' object cannot be converted to 'SchemaSerializer'`. This is a 500 error in any process.
+
+A different solution removed the cost. `django.setup()` no longer imports `posthog.schema` at all.
+
+_Also asked as:_ `defer_build`, make the schema import lazy, pydantic model build is slow at startup
+
+### Replace pydantic in the generated schema with plain dataclasses
+
+**Verdict: not viable today**
+
+`posthog/schema.py` has more than 1,000 generated classes. `hogli build:schema` generates the file from the TypeScript types with pydantic tooling.
+Approximately 220 files call `model_validate`, and more call the `model_validate_json` and `model_validate_python` variants. The API layer depends on this validation. Thus a change of the model library is not a local change.
+
+The import cost is solved. `posthog.schema` costs approximately 2 seconds to import, but `django.setup()` no longer loads it. The enums also moved to `posthog.schema_enums`, which imports in approximately 20 ms.
+Import a model from `posthog.schema` inside the method that uses it. Take the enums from `posthog.schema_enums`.
+
+_Also asked as:_ remove pydantic, use dataclasses for the schema, the schema import is slow, why is `posthog.schema` so big
+
+## Database migrations
+
+### Switch the Person model to the partitioned table with a Django setting
+
+**Verdict: eight attempts closed unmerged** · Nov 2025 · [#41436](https://github.com/PostHog/posthog/pull/41436), [#41513](https://github.com/PostHog/posthog/pull/41513), [#41522](https://github.com/PostHog/posthog/pull/41522), [#41600](https://github.com/PostHog/posthog/pull/41600), [#41604](https://github.com/PostHog/posthog/pull/41604), [#41669](https://github.com/PostHog/posthog/pull/41669), [#41698](https://github.com/PostHog/posthog/pull/41698), [#41813](https://github.com/PostHog/posthog/pull/41813)
+
+The goal is to move the Person model from `posthog_person` to a table that is partitioned by `team_id`. Eight PRs tried five mechanisms.
+
+A `PERSON_TABLE_NAME` setting that gives `db_table` to the model. A dual manager that reads both tables and prefers the new one. A swap of the two table names in the database. A wrapper that rejects any query to a partitioned table without `team_id` in the `WHERE` clause. A separate test database for the person tables.
+
+None of them merged. The author wrote this on [#41513](https://github.com/PostHog/posthog/pull/41513):
+
+> I'm still not sure if I got on the wrong track here by wanting to bend all test setup to use the person_new table and other sqlx migrated stuff. It seems I overlooked something fundamental since things are failing so much.
+
+One narrow PR from the same window did merge. [#41620](https://github.com/PostHog/posthog/pull/41620) put `team_id` into the Person queries that lacked it, and it carried the `PERSON_TABLE_NAME` setting to master.
+The setting is in `posthog/settings/data_stores.py`, and `Person.Meta.db_table` reads it. It defaults to `posthog_person`, so the cutover is off.
+The switch exists. The eight PRs above failed at what surrounds it: the test setup, the dual reads, and the partition guard.
+`posthog_person_new` comes from the sqlx migrations in `rust/persons_migrations/`. Three Dagster jobs read it, and one carries a comment about a future name swap.
+Person and group data now goes through the gRPC client in `posthog/personhog_client/`. `AGENTS.md` makes that client the required interface and prohibits new ORM queries against the person tables.
+
+Read this entry before you propose a Django-level cutover. The setting is already there.
+What is missing is everything that must be true before a person changes its value.
+
+_Also asked as:_ partition the person table, `PERSON_TABLE_NAME`, `posthog_person_new`, dual-table reads, cut over the Person model
+
+## API contracts
+
+### Validate the API responses against the generated OpenAPI schema
+
+**Verdict: six attempts, none merged** · Mar 2026 to Jun 2026
+
+The idea returns in two shapes.
+
+End-to-end traffic validation, in the Playwright run. Most of these use a Django middleware, and [#49895](https://github.com/PostHog/posthog/pull/49895) uses Spectral and a Prism proxy instead: [#49898](https://github.com/PostHog/posthog/pull/49898), [#49932](https://github.com/PostHog/posthog/pull/49932), [#49940](https://github.com/PostHog/posthog/pull/49940).
+Response validation inside the pytest run, with a report as a CI artifact: [#56804](https://github.com/PostHog/posthog/pull/56804), [#56810](https://github.com/PostHog/posthog/pull/56810).
+
+Each PR made the validation optional and non-blocking, to avoid noise. None of them merged.
+Read this history before you start a seventh attempt. Six PRs that all stop before the merge is a signal about the design, not about the effort.
+
+The generated types have a different guard today. The serializers produce the OpenAPI schema, and `hogli build:openapi` generates the TypeScript from it. CI fails when the committed output does not match.
+
+_Also asked as:_ contract testing, validate responses against the schema, spectral, prism, schema drift in CI
+
+## Product isolation
+
+### Move `ee/` into `products/enterprise/backend/`
+
+**Verdict: rejected** · Nov 2025 · [#41025](https://github.com/PostHog/posthog/pull/41025)
+
+The PR moved 613 files and kept the git history. It kept the app label `ee`, so the database did not change. Django validated, and no migration was necessary.
+
+The PR did not merge, and `ee/` is still a top-level directory.
+
+A mechanically correct move is not sufficient for a directory of this size. If you propose this again, say who reviews 613 moved files, and what breaks for each open PR that touches `ee/`.
+
+_Also asked as:_ move ee to products, get rid of the ee folder, enterprise product folder
+
+### Use `logs` as the first product behind a facade
+
+**Verdict: superseded** · Jun 2026 · [#63184](https://github.com/PostHog/posthog/pull/63184)
+
+`logs` was the first candidate for the field test. Core imports its models, its query runner, its celery task, and its temporal wiring. Thus `logs` is a difficult example.
+
+The field test moved to `web_analytics` in [#63535](https://github.com/PostHog/posthog/pull/63535). [#63193](https://github.com/PostHog/posthog/pull/63193) collected the tools and the skill.
+The note at the closure says that the tools can isolate `logs` quickly, if the team still wants this.
+The doctrine from this work came later, in [#71486](https://github.com/PostHog/posthog/pull/71486).
+
+_Also asked as:_ which product should we isolate first, facade migration example, isolate logs
+
+## PR structure
+
+### Put two closely coupled layers in two stacked PRs
+
+**Verdict: rejected for this change** · Aug 2026 · [#87643](https://github.com/PostHog/posthog/pull/87643), merged into [#87644](https://github.com/PostHog/posthog/pull/87644)
+
+The two layers read well as two stories. They did not work as two diffs.
+Both layers changed the same four modules, and sometimes the same lines.
+The upper layer deleted a block that the lower layer corrected. It also changed the signature of a function that the lower layer divided.
+
+Each correction in the lower layer needed a repeat through these collisions. The repeat occurs again at each restack.
+
+Divide a change by its diff surface, not by its story. If two layers touch the same lines, one PR needs less review effort than two.
+
+_Also asked as:_ should I stack these, split this PR, break the change into reviewable layers
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 5a202911c83f..343381087cc4 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -661,9 +661,9 @@ snapshots:
components-hogcharts-barchart--with-value-labels--light:
hash: v1.k794b7964.6aff50b1c147df21230d149c147deb023f6308cd2e01983b58e65461dd4c7207.vLZ6XqXFhSTA3hr10RUEv9vzOgxKWYtUzWj0OvoCTU0
components-hogcharts-boxplot--multi-series-grouped--dark:
- hash: v1.k794b7964.100052bfc106ff42b501157a182699d6814f90be488c4e03ff6d7476fdd05cbe.bLaFB-wXyDHV1bbsR0ssdrE6qYaPmFcUTXnYzWDasY0
+ hash: v1.k794b7964.fd8a8c11e35ef420a663807a21a6349eba0ae2658c6c3d114763f4629dd45934.d23KBypemM2QY55g8_QWVUc9diJoAL1oNZjK3d3Ebs4
components-hogcharts-boxplot--multi-series-grouped--light:
- hash: v1.k794b7964.54b13669ffe5a6341141893094ff6887b677688ab89242bc82217f54129fa7a3.TjWyBhlbZbDBnspp8YhHvJCM5Bu146okpGeZRzWrNKs
+ hash: v1.k794b7964.016f9887360412c0b22e1b45e677ab2c289961ce86382e4bb7287f7d8c5dd110.cuv1S6OAl-253E0RJgLwIzHtVxH1ji5mzIOT3GTBWnk
components-hogcharts-boxplot--no-grid--dark:
hash: v1.k794b7964.ec49b9672a800158cc0dd7f1a517dfec0912d03ce7bc8b56c9a08492fdef723f.5tbdsdN5AB0mtLjoNobhw1skLsS4m5Sc6Es7TSdhTWg
components-hogcharts-boxplot--no-grid--light:
@@ -2689,9 +2689,9 @@ snapshots:
insights-boldnumber--empty-result--light:
hash: v1.k794b7964.2d0c17643991833246321ec917a7b3a49228fb7b217b6bdb8742bbebe1164a5d.bFD6Zu0IU06yXEaw5GNlDjBhvFh8vqBKvXRQwCS53gs
insights-boxplot--default--dark:
- hash: v1.k794b7964.11bcd18191a129b7b821de0f214b9d5fe9169c98ac2d809c636f87648943d802.DWBO44Qbzd87d8axYy5zm4MSecWPDXxD1I3ZTUUHuUc
+ hash: v1.k794b7964.0703a7022dee7396b3f005ebe10aeb376345086ef4c7fc2458108ce87351123d.Ps1FFHDhHcNR2ApWX_ypYngS_xpINJtwB9EH-9n8vLU
insights-boxplot--default--light:
- hash: v1.k794b7964.80daaf49ad0c7b82eb9071a207eeadd6db945e06743ba4b64690613f5b70265c.nDmyP88KGm6rnkO-8qNwFQM205eMFRUngrx5R4BjIws
+ hash: v1.k794b7964.0e1fc3799ec1876e6ccca4da11c8ebc63836acbe5cbe7453912a385be0564dc5.Pxd_3RMZ6szO03ZwKl6_6UT1AW7TiK_1WllTaLSn60k
insights-funnelbarhorizontalchart--breakdown--dark:
hash: v1.k794b7964.e1339725b2bf6fd5c152865170f77320dbc22dc941a23fdc81694f3a4b17f884.poYcV8iWzKuJI0od7oM9G9gjUXrgP38w7bGlQushUjg
insights-funnelbarhorizontalchart--breakdown--light:
@@ -7132,6 +7132,14 @@ snapshots:
hash: v1.k794b7964.e0482531b63e3c6c58bb438b0848b476db1ebadb82736444afb0004e5a7d9a2e.EgJaf86aq6hyYkPzdF8Aydv7rycYiRr_UV8fYh_LIL0
scenes-app-insights-side-panel-actions--unsaved-insight--light:
hash: v1.k794b7964.2b3ed2b5ffec763b0f7e13af74c9e9d57df96464976ae6014512a0d7def42ad3.paLp2yPHgkf2-ncauikOAUdR87k6-KhAh-Nm4uyQ380
+ scenes-app-insights-sqlboxplot--edit-options--dark:
+ hash: v1.k794b7964.70df9d8ffa57acac2d95f47d3298bb62ca009f1f4b556717ee0bbe319ab7754e.VwWfKL8TJDDpSmrV3mWOD0VkvzSnBrYJTFtSRpxlQcw
+ scenes-app-insights-sqlboxplot--edit-options--light:
+ hash: v1.k794b7964.836958043bd3b8203146f802f75a29f5edb8ad4c056a5e2cd01d75a14fa453bb.lDduPTYwYFGPpa1wZHBJEHrKfyxy9jVNMI9-pUQAAjE
+ scenes-app-insights-sqlboxplot--grouped-series--dark:
+ hash: v1.k794b7964.0062f9e095cb6266619d2886be6973be68c7fc906f50d8b37064390de6757656.frURXaRskPxT8wSqipNvLkFVNmgq3uiSKORwD_fLWDA
+ scenes-app-insights-sqlboxplot--grouped-series--light:
+ hash: v1.k794b7964.d6214c2e48859cedc4c7dc30552657668a7e9a1a7f92f4643b3d2dbe56df2d57.ytDzXwguLj-27G1jRP28BG-bv7d6q1z4CgNsAmbpqgw
scenes-app-insights-sqllinechart--sql-bar-chart-value-labels-quill--dark:
hash: v1.k794b7964.482f71c0aadde604594294d6e723590838e5b47f543adf3baf39c5ad2f2a9c3e.bpFdUZ6dFKKinvbM5jklQHy3pQoeURgjrWtjcXipKBw
scenes-app-insights-sqllinechart--sql-bar-chart-value-labels-quill--light:
@@ -7945,9 +7953,9 @@ snapshots:
scenes-app-settings-environment--settings-environment-error-tracking--light:
hash: v1.k794b7964.5ac167271f00184ce6159d0072ce163542658b08419f6e70b5cf6a6ca201a718.X6dKpM6uGw_g_FAwdQ5PukTVcKWGAa77bMOzgw-kiXY
scenes-app-settings-environment--settings-environment-error-tracking-configuration--dark:
- hash: v1.k794b7964.033e5499811363c36dee07950ff09f33413e64b81c8522859c85748bc0443e3f.Hz66ItOfiG51KAAT2yidV4lINJmLINi-9_k81Ygdk80
+ hash: v1.k794b7964.3d7a3006130bdc66e1456cb81566e21bb2179f19d17caf91ead7a41ba4be9234.tbIKl_Tlb1T7RBNR3Ic9n51t6kNpTkGsw28sXZHW-Gs
scenes-app-settings-environment--settings-environment-error-tracking-configuration--light:
- hash: v1.k794b7964.56f8265ff5de3733a4d00b0dbde7c0fff36ccdc886f713881134159802a13d39.9ArXuAkoxa6dyVQ_uspbjHWKM8YIgCPgCbDUnLHsxso
+ hash: v1.k794b7964.7d62fa47841096cceaa13c9a7cb15c1152b2f929ee030674bc0d97c5bc66c239.GptZ3dbdhoABnDP-dWhaqUCQOTeKYOsMFokkyWuNjwQ
scenes-app-settings-environment--settings-environment-feature-flags--dark:
hash: v1.k794b7964.56670f556ffac0360ec521741db9822f4e545e25b45bd8ed4ad012da8885c30a.qj9dLuAFy8v0TwEH-UTymXnlWPe2eu-YjAOpe2zc3Q8
scenes-app-settings-environment--settings-environment-feature-flags--light:
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index dfb9afbd5bf9..e75cbd14098d 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -203,6 +203,7 @@ export const FEATURE_FLAGS = {
REPLAY_EXCLUDE_FROM_HIDE_RECORDINGS_MENU: 'replay-exclude-from-hide-recordings-menu', // owner: #team-replay, used to exclude what other people are seeing in Replay
SHOW_UPGRADE_TO_MANAGED_ACCOUNT: 'show-upgrade-to-managed-account', // owner: #team-billing, used to give free accounts a way to force upgrade to managed account
WEBHOOKS_DENYLIST: 'webhooks-denylist', // owner: #team-ingestion, used to disable webhooks for certain companies
+ SQL_BOX_PLOT_INSIGHT: 'sql-box-plot-insight', // owner: @pauldambra #team-product-analytics
// Legacy flags, TBD if they need to be removed
BATCH_EXPORTS_POSTHOG_HTTP: 'posthog-http-batch-exports', // owner: #team-batch-exports
diff --git a/frontend/src/mocks/fixtures/api/projects/team_id/insights/sqlBoxPlot.json b/frontend/src/mocks/fixtures/api/projects/team_id/insights/sqlBoxPlot.json
new file mode 100644
index 000000000000..b0bf619e33c8
--- /dev/null
+++ b/frontend/src/mocks/fixtures/api/projects/team_id/insights/sqlBoxPlot.json
@@ -0,0 +1,208 @@
+{
+ "id": 31,
+ "short_id": "boxPlot",
+ "name": "Latency distribution by plan",
+ "derived_name": null,
+ "filters": {},
+ "query": {
+ "kind": "DataVisualizationNode",
+ "source": {
+ "kind": "HogQLQuery",
+ "query": "SELECT\n toStartOfWeek(timestamp) AS bucket,\n properties.plan AS series,\n min(toFloat(properties.latency)) AS min,\n quantile(0.25)(toFloat(properties.latency)) AS p25,\n quantile(0.5)(toFloat(properties.latency)) AS median,\n avg(toFloat(properties.latency)) AS mean,\n quantile(0.75)(toFloat(properties.latency)) AS p75,\n max(toFloat(properties.latency)) AS max\nFROM events\nGROUP BY bucket, series\nORDER BY bucket, series"
+ },
+ "display": "BoxPlot",
+ "chartSettings": {
+ "boxPlot": {
+ "xAxisColumn": "bucket",
+ "seriesColumn": "series",
+ "minColumn": "min",
+ "p25Column": "p25",
+ "medianColumn": "median",
+ "meanColumn": "mean",
+ "p75Column": "p75",
+ "maxColumn": "max",
+ "excludeOutliers": true
+ },
+ "showLegend": true,
+ "leftYAxisSettings": {
+ "label": "Latency (ms)",
+ "showGridLines": true
+ }
+ }
+ },
+ "order": null,
+ "deleted": false,
+ "dashboards": [],
+ "dashboard_tiles": [],
+ "last_refresh": "2026-03-04T16:10:59.052768Z",
+ "cache_target_age": "2026-03-04T22:10:59.052768Z",
+ "next_allowed_client_refresh": "2026-03-04T16:11:59.052768Z",
+ "result": [
+ [
+ "2026-01-05",
+ "Free",
+ 80,
+ 120,
+ 170,
+ 195,
+ 240,
+ 620
+ ],
+ [
+ "2026-01-05",
+ "Paid",
+ 45,
+ 70,
+ 95,
+ 110,
+ 135,
+ 310
+ ],
+ [
+ "2026-01-12",
+ "Free",
+ 75,
+ 115,
+ 165,
+ 185,
+ 225,
+ 560
+ ],
+ [
+ "2026-01-12",
+ "Paid",
+ 40,
+ 65,
+ 88,
+ 102,
+ 125,
+ 280
+ ],
+ [
+ "2026-01-19",
+ "Free",
+ 90,
+ 135,
+ 190,
+ 215,
+ 265,
+ 710
+ ],
+ [
+ "2026-01-19",
+ "Paid",
+ 50,
+ 75,
+ 105,
+ 118,
+ 145,
+ 340
+ ],
+ [
+ "2026-01-26",
+ "Free",
+ 70,
+ 105,
+ 150,
+ 172,
+ 210,
+ 490
+ ],
+ [
+ "2026-01-26",
+ "Paid",
+ 38,
+ 60,
+ 82,
+ 96,
+ 118,
+ 250
+ ]
+ ],
+ "hasMore": false,
+ "columns": [
+ "bucket",
+ "series",
+ "min",
+ "p25",
+ "median",
+ "mean",
+ "p75",
+ "max"
+ ],
+ "created_at": "2026-03-04T16:11:08.460573Z",
+ "created_by": {
+ "id": 1,
+ "uuid": "019c66e6-fd77-0000-a6bb-509ac38f988e",
+ "distinct_id": "otSl3FZiAxw3Vv8GUTeWtZcV3mdpR0lH9pprZmBRKXd",
+ "first_name": "Employee 427",
+ "last_name": "",
+ "email": "test@posthog.com",
+ "is_email_verified": null,
+ "hedgehog_config": null,
+ "role_at_organization": "engineering"
+ },
+ "description": null,
+ "updated_at": "2026-03-04T16:11:08.460609Z",
+ "favorited": false,
+ "saved": true,
+ "last_modified_at": "2026-03-04T16:11:08.456506Z",
+ "last_modified_by": {
+ "id": 1,
+ "uuid": "019c66e6-fd77-0000-a6bb-509ac38f988e",
+ "distinct_id": "otSl3FZiAxw3Vv8GUTeWtZcV3mdpR0lH9pprZmBRKXd",
+ "first_name": "Employee 427",
+ "last_name": "",
+ "email": "test@posthog.com",
+ "is_email_verified": null,
+ "hedgehog_config": null,
+ "role_at_organization": "engineering"
+ },
+ "is_sample": false,
+ "effective_restriction_level": 21,
+ "effective_privilege_level": 37,
+ "user_access_level": "manager",
+ "timezone": "UTC",
+ "is_cached": true,
+ "query_status": null,
+ "hogql": "SELECT\n toStartOfWeek(timestamp) AS bucket,\n properties.plan AS series,\n min(toFloat(properties.latency)) AS min,\n quantile(0.25)(toFloat(properties.latency)) AS p25,\n quantile(0.5)(toFloat(properties.latency)) AS median,\n avg(toFloat(properties.latency)) AS mean,\n quantile(0.75)(toFloat(properties.latency)) AS p75,\n max(toFloat(properties.latency)) AS max\nFROM events\nGROUP BY bucket, series\nORDER BY bucket, series\nLIMIT 101\nOFFSET 0",
+ "types": [
+ [
+ "bucket",
+ "Date"
+ ],
+ [
+ "series",
+ "String"
+ ],
+ [
+ "min",
+ "Float64"
+ ],
+ [
+ "p25",
+ "Float64"
+ ],
+ [
+ "median",
+ "Float64"
+ ],
+ [
+ "mean",
+ "Float64"
+ ],
+ [
+ "p75",
+ "Float64"
+ ],
+ [
+ "max",
+ "Float64"
+ ]
+ ],
+ "resolved_date_range": null,
+ "alerts": [],
+ "last_viewed_at": "2026-03-04T16:11:30.018817Z",
+ "tags": [],
+ "filters_hash": "cache_1_eb04b61fff884f29a4c74600aea3684fb7769ff1fa6e0d4e59865ee187c3b58b"
+}
diff --git a/frontend/src/queries/nodes/DataVisualization/Components/BoxPlotSeriesTab.test.tsx b/frontend/src/queries/nodes/DataVisualization/Components/BoxPlotSeriesTab.test.tsx
new file mode 100644
index 000000000000..c9ed5f94f432
--- /dev/null
+++ b/frontend/src/queries/nodes/DataVisualization/Components/BoxPlotSeriesTab.test.tsx
@@ -0,0 +1,107 @@
+import '@testing-library/jest-dom'
+
+import { render, screen, waitFor } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { BindLogic } from 'kea'
+
+import { DataVisualizationNode, HogQLQueryResponse, NodeKind } from '~/queries/schema/schema-general'
+import { initKeaTests } from '~/test/init'
+import { ChartDisplayType } from '~/types'
+
+import { dataNodeLogic } from '../../DataNode/dataNodeLogic'
+import { DataVisualizationLogicProps, dataVisualizationLogic } from '../dataVisualizationLogic'
+import { BoxPlotSeriesTab } from './BoxPlotSeriesTab'
+
+const query: DataVisualizationNode = {
+ kind: NodeKind.DataVisualizationNode,
+ source: { kind: NodeKind.HogQLQuery, query: 'select * from summaries' },
+ display: ChartDisplayType.BoxPlot,
+ chartSettings: {
+ boxPlot: {
+ xAxisColumn: 'bucket',
+ minColumn: 'min',
+ p25Column: 'p25',
+ medianColumn: 'median',
+ meanColumn: 'mean',
+ p75Column: 'p75',
+ maxColumn: 'max',
+ },
+ },
+}
+
+const cachedResults: HogQLQueryResponse = {
+ results: [['Mon', 1, 2, 3, 4, 5, 6, 0]],
+ columns: ['bucket', 'min', 'p25', 'median', 'mean', 'p75', 'max', 'alternate_min'],
+ types: [
+ ['bucket', 'String'],
+ ['min', 'Float64'],
+ ['p25', 'Float64'],
+ ['median', 'Float64'],
+ ['mean', 'Float64'],
+ ['p75', 'Float64'],
+ ['max', 'Float64'],
+ ['alternate_min', 'Float64'],
+ ],
+}
+
+describe('BoxPlotSeriesTab', () => {
+ it('shows box plot roles and saves a changed statistic column', async () => {
+ initKeaTests()
+ const setQuery = jest.fn()
+ let currentQuery = query
+ const props: DataVisualizationLogicProps = {
+ key: 'box-plot-series-tab',
+ query: currentQuery,
+ cachedResults,
+ dataNodeCollectionId: 'box-plot-series-tab',
+ setQuery: (setter) => {
+ currentQuery = setter(currentQuery)
+ setQuery(currentQuery)
+ },
+ }
+
+ dataNodeLogic({
+ key: props.key,
+ query: query.source,
+ cachedResults,
+ dataNodeCollectionId: props.dataNodeCollectionId,
+ }).mount()
+ dataVisualizationLogic(props).mount()
+
+ const { container } = render(
+
+
+
+ )
+
+ expect(screen.getByText('25th percentile')).toBeInTheDocument()
+ expect(screen.getByText('75th percentile')).toBeInTheDocument()
+
+ const minimumSelect = container.querySelector('[data-attr="box-plot-minColumn"]')
+ if (!(minimumSelect instanceof HTMLElement)) {
+ throw new Error('Expected the minimum column selector')
+ }
+
+ const user = userEvent.setup()
+ await user.click(minimumSelect)
+ await user.click(await screen.findByText('alternate_min'))
+
+ await waitFor(() => expect(currentQuery.chartSettings?.boxPlot?.minColumn).toBe('alternate_min'))
+
+ const xAxisSelect = container.querySelector('[data-attr="box-plot-x-axis-column"]')
+ if (!(xAxisSelect instanceof HTMLElement)) {
+ throw new Error('Expected the X-axis column selector')
+ }
+
+ await user.click(xAxisSelect)
+ const noneOption = (await screen.findAllByText('None')).find(
+ (element) => !element.closest('[data-attr="box-plot-series-column"]')
+ )
+ if (!noneOption) {
+ throw new Error('Expected the None option')
+ }
+ await user.click(noneOption)
+
+ await waitFor(() => expect(currentQuery.chartSettings?.boxPlot?.xAxisColumn).toBeNull())
+ })
+})
diff --git a/frontend/src/queries/nodes/DataVisualization/Components/BoxPlotSeriesTab.tsx b/frontend/src/queries/nodes/DataVisualization/Components/BoxPlotSeriesTab.tsx
new file mode 100644
index 000000000000..1c1626b68e01
--- /dev/null
+++ b/frontend/src/queries/nodes/DataVisualization/Components/BoxPlotSeriesTab.tsx
@@ -0,0 +1,88 @@
+import { useActions, useValues } from 'kea'
+
+import { LemonBanner, LemonLabel, LemonSelect, LemonTag } from '@posthog/lemon-ui'
+
+import { BoxPlotSettings } from '~/queries/schema/schema-general'
+
+import { Column, dataVisualizationLogic } from '../dataVisualizationLogic'
+import { BOX_PLOT_STATISTICS } from './Charts/sqlBoxPlotAdapter'
+
+const NONE_COLUMN = '__posthog_box_plot_none__'
+
+export const BoxPlotSeriesTab = (): JSX.Element => {
+ const { chartSettings, columns, numericalColumns, responseLoading } = useValues(dataVisualizationLogic)
+ const { updateChartSettings } = useActions(dataVisualizationLogic)
+ const settings = chartSettings.boxPlot ?? {}
+
+ const updateSettings = (updates: Partial): void => {
+ updateChartSettings({ boxPlot: { ...settings, ...updates } })
+ }
+
+ const toColumnOption = ({ name, type }: Column): { value: string; label: JSX.Element } => ({
+ value: name,
+ label: (
+