From 90c1ab63b4c02baddfffe8aae9f2ad765784571c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 06:41:46 -0700 Subject: [PATCH 01/11] fix(maintainer): verify terminal E2E coordinator evidence Signed-off-by: Carlos Villela --- .../scripts/check-gates.ts | 1116 ++++++++++++- .../check-gates-coordinator-evidence.test.ts | 1430 +++++++++++++++++ test/skills/check-gates-evidence.test.ts | 10 +- test/skills/check-gates-test-fixtures.ts | 337 +++- 4 files changed, 2792 insertions(+), 101 deletions(-) create mode 100644 test/skills/check-gates-coordinator-evidence.test.ts diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index 3aeda111c2b..66cba78f0db 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -10,6 +10,7 @@ * Usage: node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts [--repo OWNER/REPO] */ +import { isDeepStrictEqual } from "node:util"; import { ghJson, isRiskyFile, @@ -316,7 +317,60 @@ interface E2eCoordinationEvidence { valid: boolean | null; startedAt?: number; completedAt?: number; + enclosingCoordinatorStartedAt?: number; + enclosingCoordinatorCompletedAt?: number; trustedCustomCheckId?: number; + checkSnapshot?: E2eCoordinationCheckSnapshot; + coordinatorSnapshot?: E2eCoordinatorInventorySnapshot; + selectedCheckId?: number; +} + +interface E2eCoordinationCheckSnapshot { + checkRuns: Array>; +} + +interface E2eCoordinatorRunMetadata { + id: number; + attempt: number; + createdAt: number; + updatedAt: number; + event: E2eCoordinatorEvent; + displayTitle: string; + headSha: string; + status: string; + conclusion: string | null; +} + +type E2eCoordinatorEvent = "workflow_run" | "workflow_dispatch"; + +interface E2eCoordinatorRunPartition { + startedAt: number; + completedAt: number; +} + +interface E2eCoordinatorInventoryRecord { + value: Record; + createdAt: number; + event: E2eCoordinatorEvent; + partitionIndex: number; +} + +interface E2eCoordinatorCandidateSnapshot { + listedRun: Record; + firstRun: Record; + jobPages: unknown[]; + refreshedRun: Record; +} + +interface E2eCoordinatorInventorySnapshot { + candidates: E2eCoordinatorCandidateSnapshot[]; +} + +interface E2eCoordinatorEvaluation { + valid: boolean | null; + snapshot?: E2eCoordinatorInventorySnapshot; + coordinateStartedAt?: number; + coordinateCompletedAt?: number; } const E2E_RETRYABLE_FAILURE_MARKER_PREFIX = "", + }, + }); +} + +const DEFAULT_COORDINATOR_RANGES = [ + "2025-12-31T00:00:00Z..2025-12-31T12:00:00Z", + "2025-12-31T12:00:00Z..2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z..2026-01-01T00:03:00Z", +] as const; + +function paginatedRunIds(ids: number[]): number[][] { + return Array.from({ length: Math.ceil(ids.length / 100) }, (_value, index) => + ids.slice(index * 100, (index + 1) * 100), + ); +} + +interface CoordinatorFixture { + additionalStatusChecks?: NonNullable; + coordinator?: Partial; + coordinatorList?: Partial; + coordinatorJobs?: ActionJobFixture[]; + coordinatorRunPages?: number[][]; + coordinatorRunPartitions?: CoordinatorRunPartitionFixture[]; + coordinationCheckPages?: unknown[]; + customCheck?: Record; + extraRuns?: Record; + finalCheckOverrides?: Record< + string, + Partial[number]> + >; + finalCoordinationCheckPages?: unknown[]; + finalFormerCoordinationCheckPages?: unknown[]; + finalPrAfterCiEvidence?: Record; + additionalChecksAfterE2eEvidence?: NonNullable; + headRepository?: string; + includeInitialSeedEvidence?: boolean; + observationTime?: string; + seedRun?: Partial; +} + +const LIVE_FORK_COORDINATOR_RANGES = [ + "2026-08-03T00:00:00Z..2026-08-03T12:00:00Z", + "2026-08-03T12:00:00Z..2026-08-04T00:00:00Z", + "2026-08-04T00:00:00Z..2026-08-04T12:00:00Z", + "2026-08-04T12:00:00Z..2026-08-04T14:58:00Z", +] as const; + +function authorizedForkLifecycleFixture( + additionalAutomaticRuns: Record = {}, +): CoordinatorFixture { + const automaticCoordinator = { + ...e2eCoordinatorRun(), + createdAt: "2026-08-04T14:36:34Z", + updatedAt: "2026-08-04T14:37:04Z", + jobs: coordinatorJobs({ + startedAt: "2026-08-04T14:36:35Z", + completedAt: "2026-08-04T14:37:03Z", + }), + }; + const manualCoordinator = { + ...e2eManualCoordinatorRun(), + createdAt: "2026-08-04T14:38:13Z", + updatedAt: "2026-08-04T14:57:43Z", + headSha: "d".repeat(40), + jobs: coordinatorJobs({ + startedAt: "2026-08-04T14:38:14Z", + completedAt: "2026-08-04T14:57:42Z", + }), + }; + const automaticRunIds = [9501, ...Object.keys(additionalAutomaticRuns).map(Number)]; + return { + headRepository: "example/fork", + observationTime: "2026-08-04T14:58:00Z", + customCheck: { + started_at: "2026-08-04T14:23:40Z", + completed_at: "2026-08-04T14:57:39Z", + }, + coordinator: manualCoordinator, + seedRun: { + headRepository: "example/fork", + createdAt: "2026-08-04T14:21:55Z", + updatedAt: "2026-08-04T14:23:47Z", + jobs: seedJobs().map((job) => ({ + ...job, + startedAt: "2026-08-04T14:23:15Z", + completedAt: "2026-08-04T14:23:42Z", + })), + }, + coordinatorRunPartitions: [ + ...LIVE_FORK_COORDINATOR_RANGES.map((createdRange, index) => ({ + createdRange, + runPages: [index === LIVE_FORK_COORDINATOR_RANGES.length - 1 ? automaticRunIds : []], + })), + { + createdRange: LIVE_FORK_COORDINATOR_RANGES.at(-1)!, + event: "workflow_dispatch", + runPages: [[9500]], + }, + ], + extraRuns: { + "90": { + ...prWorkflowRun( + "success", + [ + { id: 1, name: "checks" }, + { id: 2, name: "changes" }, + ], + true, + ), + headRepository: "example/fork", + pullRequests: [], + }, + "9501": automaticCoordinator, + ...additionalAutomaticRuns, + }, + }; +} + +function delayedRetryFixture(): CoordinatorFixture { + return { + seedRun: { + createdAt: "2026-08-04T14:07:32Z", + updatedAt: "2026-08-04T14:08:01Z", + jobs: seedJobs().map((job) => ({ + ...job, + startedAt: "2026-08-04T14:07:40Z", + completedAt: "2026-08-04T14:08:00Z", + })), + }, + customCheck: { + started_at: "2026-08-04T14:28:06Z", + completed_at: "2026-08-04T14:38:01Z", + }, + coordinator: { + createdAt: "2026-08-04T14:27:44Z", + updatedAt: "2026-08-04T14:38:06Z", + }, + coordinatorJobs: coordinatorJobs({ + startedAt: "2026-08-04T14:27:47Z", + completedAt: "2026-08-04T14:38:05Z", + }), + coordinatorRunPartitions: [ + { + createdRange: "2026-08-03T00:00:00Z..2026-08-03T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2026-08-03T12:00:00Z..2026-08-04T00:00:00Z", + runPages: [[]], + }, + { + createdRange: "2026-08-04T00:00:00Z..2026-08-04T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2026-08-04T12:00:00Z..2026-08-04T14:38:01Z", + runPages: [[9500]], + }, + ], + }; +} + +function seedStatusChecks( + runId: number, + jobs: ActionJobFixture[], + fallbackStartedAt = "2026-01-01T00:01:00Z", +) { + return jobs.map((job) => ({ + __typename: "CheckRun", + name: job.name, + workflowName: "E2E / PR Gate Controller", + detailsUrl: `https://github.com/NVIDIA/NemoClaw/actions/runs/${runId}/job/${job.id}`, + startedAt: job.startedAt ?? fallbackStartedAt, + status: (job.status ?? "completed").toUpperCase(), + ...(job.conclusion === null ? {} : { conclusion: (job.conclusion ?? "success").toUpperCase() }), + })); +} + +function manualOnlyCoordinatorPartitions(): CoordinatorRunPartitionFixture[] { + return DEFAULT_COORDINATOR_RANGES.map((createdRange, index) => ({ + createdRange, + event: "workflow_dispatch", + runPages: [index === DEFAULT_COORDINATOR_RANGES.length - 1 ? [9500] : []], + })); +} + +function runGateWithCoordinator({ + additionalStatusChecks = [], + coordinator = {}, + coordinatorList, + coordinatorJobs: configuredJobs, + coordinatorRunPages = [[9500]], + coordinatorRunPartitions, + coordinationCheckPages, + customCheck = {}, + extraRuns = {}, + finalCheckOverrides = {}, + finalCoordinationCheckPages, + finalFormerCoordinationCheckPages, + finalPrAfterCiEvidence, + additionalChecksAfterE2eEvidence, + headRepository, + includeInitialSeedEvidence = true, + observationTime, + seedRun = {}, +}: CoordinatorFixture = {}) { + const defaultCoordinator = e2eCoordinatorRun(); + const jobs = seedRun.jobs ?? seedJobs(); + const statusChecks = [ + ...successfulRequiredChecks().filter((check) => check.name !== "initialize"), + ...(includeInitialSeedEvidence ? seedStatusChecks(407, jobs, seedRun.createdAt) : []), + ...additionalStatusChecks, + ]; + const finalStatusChecks = + Object.keys(finalCheckOverrides).length > 0 + ? statusChecks.map((check) => ({ + ...check, + ...finalCheckOverrides[check.name], + })) + : undefined; + const coordinatorJobs = configuredJobs ?? coordinator.jobs ?? defaultCoordinator.jobs; + return runGate({ + body: "Signed-off-by: Example User ", + verified: true, + headRepository, + observationTime, + statusChecks, + coordinatorRunPages, + coordinatorRunPartitions, + coordinatorListAttempts: coordinatorList + ? { + "9500": { + ...defaultCoordinator, + ...coordinator, + ...coordinatorList, + jobs: coordinatorJobs, + }, + } + : undefined, + coordinationCheckPages: coordinationCheckPages ?? [ + { + total_count: 1, + check_runs: [coordinationCheck(customCheck)], + }, + ], + finalCoordinationCheckPages, + finalFormerCoordinationCheckPages, + finalPr: finalStatusChecks ? { statusCheckRollup: finalStatusChecks } : undefined, + finalPrAfterCiEvidence: additionalChecksAfterE2eEvidence + ? { + ...finalPrAfterCiEvidence, + statusCheckRollup: [ + ...(finalStatusChecks ?? statusChecks), + ...additionalChecksAfterE2eEvidence, + ], + } + : finalPrAfterCiEvidence, + actionRunAttempts: { + "9500": { + ...defaultCoordinator, + ...coordinator, + jobs: coordinatorJobs, + }, + "407": { + ...exactDiffGateRun("success", jobs), + createdAt: "2026-01-01T00:01:00Z", + updatedAt: "2026-01-01T00:01:31Z", + pullRequests: [], + ...seedRun, + jobs, + }, + ...extraRuns, + }, + }); +} + +describe("maintainer merge-gate E2E coordinator evidence", () => { + it("accepts a custom check completed after its seed run when the repository coordinator encloses completion", () => { + const result = runGateWithCoordinator(); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("rejects a manual coordinator for a fork with no automatic predecessor", () => { + const manualCoordinator = { + ...e2eManualCoordinatorRun(), + headSha: "d".repeat(40), + }; + const result = runGateWithCoordinator({ + headRepository: "example/fork", + coordinator: manualCoordinator, + seedRun: { headRepository: "example/fork" }, + coordinatorRunPartitions: manualOnlyCoordinatorPartitions(), + extraRuns: { + "90": { + ...prWorkflowRun( + "success", + [ + { id: 1, name: "checks" }, + { id: 2, name: "changes" }, + ], + true, + ), + headRepository: "example/fork", + pullRequests: [], + }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("rejects a manual coordinator for a same-repository revision", () => { + const result = runGateWithCoordinator({ + coordinator: e2eManualCoordinatorRun(), + coordinatorRunPartitions: manualOnlyCoordinatorPartitions(), + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("accepts a fork check completed by manual authorization after automatic coordination", () => { + const result = runGateWithCoordinator(authorizedForkLifecycleFixture()); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("rejects a successful custom check without initial E2E seed evidence in the status rollup", () => { + const result = runGateWithCoordinator({ includeInitialSeedEvidence: false }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("rejects two initial E2E seed runs in the status rollup", () => { + const duplicateSeedJobs = seedJobs().map((job) => ({ ...job, id: job.id + 100 })); + const result = runGateWithCoordinator({ + additionalStatusChecks: seedStatusChecks(408, duplicateSeedJobs).filter( + (check) => check.name === "initialize", + ), + extraRuns: { + "408": { + ...exactDiffGateRun("success", duplicateSeedJobs), + createdAt: "2026-01-01T00:01:00Z", + updatedAt: "2026-01-01T00:01:31Z", + pullRequests: [], + }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("rejects two authenticated initial E2E seed runs with different creation times and overlapping initialize windows", () => { + const staggeredSeedJobs = seedJobs().map((job) => ({ + ...job, + id: job.id + 200, + startedAt: "2026-01-01T00:01:01Z", + completedAt: "2026-01-01T00:01:31Z", + })); + const result = runGateWithCoordinator({ + additionalStatusChecks: seedStatusChecks(409, staggeredSeedJobs).filter( + (check) => check.name === "initialize", + ), + extraRuns: { + "409": { + ...exactDiffGateRun("success", staggeredSeedJobs), + createdAt: "2026-01-01T00:01:01Z", + updatedAt: "2026-01-01T00:01:31Z", + pullRequests: [], + }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("rejects two automatic authorization predecessors for one manual coordinator", () => { + const result = runGateWithCoordinator( + authorizedForkLifecycleFixture({ + "9502": { + ...e2eCoordinatorRun(), + createdAt: "2026-08-04T14:37:05Z", + updatedAt: "2026-08-04T14:37:34Z", + jobs: coordinatorJobs({ + startedAt: "2026-08-04T14:37:06Z", + completedAt: "2026-08-04T14:37:33Z", + }), + }, + }), + ); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("accepts the exact coordinator from a later complete workflow-runs page", () => { + const unrelatedRun = { + ...e2eCoordinatorRun(), + displayTitle: "E2E Gate coordinate from unrelated CI", + }; + const result = runGateWithCoordinator({ + coordinatorRunPages: [[9501], [9500]], + extraRuns: { "9501": unrelatedRun }, + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("accepts one enclosing coordinator with older same-revision coordinator history", () => { + const result = runGateWithCoordinator({ + coordinatorRunPages: [[9501, 9500]], + extraRuns: { "9501": historicalCoordinatorRun() }, + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("accepts a retry coordinator created before the current custom check", () => { + const result = runGateWithCoordinator({ + coordinator: { createdAt: "2026-01-01T00:01:20Z" }, + coordinatorJobs: coordinatorJobs({ + startedAt: "2026-01-01T00:01:25Z", + }), + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("accepts a retry check created after its authenticated seed run finishes", () => { + const fixture = delayedRetryFixture(); + const result = runGateWithCoordinator({ + ...fixture, + coordinationCheckPages: [ + { + total_count: 2, + check_runs: [ + retryableFailure(7999, "2026-08-04T14:07:50Z", "2026-08-04T14:08:00Z"), + coordinationCheck(fixture.customCheck), + ], + }, + ], + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("rejects a stale seed with no predecessor for a newer coordination check", () => { + const result = runGateWithCoordinator(delayedRetryFixture()); + + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "initialize: latest attempt evidence incomplete", + ); + }); + + it("rejects a seed run without observable initialize timing", () => { + const result = runGateWithCoordinator({ + seedRun: { + jobs: seedJobs().map((job) => + job.name === "initialize" ? { ...job, omitStartedAt: true } : job, + ), + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ allPass: false, gates: { ci: { pass: false } } }); + expect(output.gates.ci.failingChecks).toContain( + "initialize: latest attempt evidence incomplete", + ); + }); + + it("rejects a seed run that encloses one check without an initialize job", () => { + const result = runGateWithCoordinator({ + seedRun: { + updatedAt: "2026-01-01T00:02:31Z", + jobs: seedJobs().filter((job) => job.name !== "initialize"), + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ allPass: false, gates: { ci: { pass: false } } }); + expect(output.gates.ci.failingChecks).toContain( + "cancel-superseded: latest attempt evidence incomplete", + ); + }); + + it("ignores a later successful seed that reuses the authenticated check", () => { + const reuseJobs = laterSeedJobs(); + const result = runGateWithCoordinator({ + additionalStatusChecks: seedStatusChecks(408, reuseJobs), + extraRuns: { + "408": { + ...exactDiffGateRun("success", reuseJobs), + createdAt: "2026-01-01T00:02:40Z", + updatedAt: "2026-01-01T00:02:51Z", + pullRequests: [], + }, + }, + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it.each([ + { + state: "failed", + run: { status: "completed", conclusion: "failure" }, + updateJobs: (jobs: ActionJobFixture[]) => + jobs.map((job) => + job.name === "cancel-superseded" ? { ...job, conclusion: "failure" } : job, + ), + }, + { + state: "pending", + run: { status: "in_progress", conclusion: null }, + updateJobs: (jobs: ActionJobFixture[]) => + jobs.map((job) => + job.name === "cancel-superseded" + ? { ...job, status: "in_progress", conclusion: null, completedAt: null } + : job, + ), + }, + ])("keeps a later $state seed reuse blocking", ({ run, updateJobs }) => { + const reuseJobs = updateJobs(laterSeedJobs()); + const result = runGateWithCoordinator({ + additionalStatusChecks: seedStatusChecks(408, reuseJobs), + extraRuns: { + "408": { + ...exactDiffGateRun(run.conclusion ?? "success", reuseJobs), + ...run, + createdAt: "2026-01-01T00:02:40Z", + updatedAt: "2026-01-01T00:02:51Z", + pullRequests: [], + }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ allPass: false, gates: { ci: { pass: false } } }); + expect(output.gates.ci.failingChecks).toContain( + "initialize: latest attempt evidence incomplete", + ); + }); + + it("finds a coordinator created before midnight for a check completed after midnight", () => { + const result = runGateWithCoordinator({ + customCheck: { + started_at: "2026-01-02T00:00:10Z", + completed_at: "2026-01-02T00:00:40Z", + }, + coordinator: { + createdAt: "2026-01-01T23:59:50Z", + updatedAt: "2026-01-02T00:00:45Z", + }, + coordinatorJobs: coordinatorJobs({ + startedAt: "2026-01-01T23:59:55Z", + completedAt: "2026-01-02T00:00:44Z", + }), + coordinatorRunPartitions: [ + { + createdRange: "2026-01-01T00:00:00Z..2026-01-01T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2026-01-01T12:00:00Z..2026-01-02T00:00:00Z", + runPages: [[9500]], + }, + { + createdRange: "2026-01-02T00:00:00Z..2026-01-02T00:00:40Z", + runPages: [[]], + }, + ], + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("accepts more than 1,000 workflow runs across complete partitions", () => { + const firstPartitionIds = Array.from({ length: 600 }, (_value, index) => 10_000 + index); + const secondPartitionIds = Array.from({ length: 600 }, (_value, index) => 20_000 + index); + const result = runGateWithCoordinator({ + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: paginatedRunIds(firstPartitionIds), + fallbackCreatedAt: "2025-12-31T06:00:00Z", + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: paginatedRunIds(secondPartitionIds), + fallbackCreatedAt: "2025-12-31T18:00:00Z", + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500]], + }, + ], + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("deduplicates an identical coordinator returned at an inclusive partition boundary", () => { + const result = runGateWithCoordinator({ + coordinator: { createdAt: "2026-01-01T00:00:00Z" }, + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: [[9500]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500]], + }, + ], + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); + + it("rejects a newer custom-check retry that appears during final observation", () => { + const result = runGateWithCoordinator({ + finalCoordinationCheckPages: [ + { + total_count: 2, + check_runs: [ + coordinationCheck(), + coordinationCheck({ + id: 8001, + status: "in_progress", + conclusion: null, + started_at: "2026-01-01T00:02:31Z", + completed_at: null, + }), + ], + }, + ], + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain("E2E / PR Gate: final evidence changed"); + }); + + it("rejects a newer queued coordinator before it creates a custom check", () => { + const result = runGateWithCoordinator({ + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500]], + finalRunPages: [[9500, 9501]], + }, + ], + extraRuns: { + "9501": { + ...e2eCoordinatorRun(), + createdAt: "2026-01-01T00:02:40Z", + updatedAt: "2026-01-01T00:02:40Z", + status: "queued", + conclusion: null, + jobs: [], + }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain("E2E / PR Gate: final evidence changed"); + }); + + it("rejects a PR revision that changes while final CI evidence is read", () => { + const result = runGateWithCoordinator({ + finalPrAfterCiEvidence: { + body: "Signed-off-by: Example User \n\nChanged during final CI.", + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { + conflicts: { + pass: false, + details: + "PR revision or merge state changed during gate evaluation; rerun the gate checker", + }, + }, + }); + }); + + it("rejects failed coordinator history from an earlier day for the same revision", () => { + const result = runGateWithCoordinator({ + coordinationCheckPages: [ + { + total_count: 2, + check_runs: [ + retryableFailure(7999, "2025-12-29T00:01:00Z", "2025-12-29T00:03:00Z"), + coordinationCheck(), + ], + }, + ], + coordinatorRunPartitions: [ + { + createdRange: "2025-12-28T00:00:00Z..2025-12-28T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-28T12:00:00Z..2025-12-29T00:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-29T00:00:00Z..2025-12-29T12:00:00Z", + runPages: [[9501]], + }, + { + createdRange: "2025-12-29T12:00:00Z..2025-12-30T00:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-30T00:00:00Z..2025-12-30T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-30T12:00:00Z..2025-12-31T00:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-31T00:00:00Z..2025-12-31T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-31T12:00:00Z..2026-01-01T00:00:00Z", + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500]], + }, + ], + extraRuns: { + "9501": { + ...e2eCoordinatorRun("failure"), + createdAt: "2025-12-29T00:02:00Z", + updatedAt: "2025-12-29T00:03:01Z", + jobs: coordinatorJobs({ + startedAt: "2025-12-29T00:02:01Z", + completedAt: "2025-12-29T00:03:00Z", + }), + }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it("rejects an extreme custom-check timestamp span before inventory traversal", () => { + const result = runGateWithCoordinator({ + customCheck: { + started_at: "2000-01-01T00:00:00Z", + completed_at: "2026-01-01T00:02:30Z", + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + it.each([ + { + field: "status", + overrides: { status: "in_progress", conclusion: null, completed_at: null }, + }, + { field: "conclusion", overrides: { conclusion: "neutral" } }, + { field: "start time", overrides: { started_at: "2026-01-01T00:01:31Z" } }, + { field: "completion time", overrides: { completed_at: "2026-01-01T00:02:31Z" } }, + { + field: "details URL", + overrides: { details_url: "https://github.com/NVIDIA/NemoClaw/runs/8001" }, + }, + ])("rejects a changed selected custom-check $field during final observation", ({ overrides }) => { + const result = runGateWithCoordinator({ + finalCoordinationCheckPages: [ + { + total_count: 1, + check_runs: [coordinationCheck(overrides)], + }, + ], + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain("E2E / PR Gate: final evidence changed"); + }); + + it("rejects changed former custom-check history during final observation", () => { + const result = runGateWithCoordinator({ + finalFormerCoordinationCheckPages: [ + { + total_count: 1, + check_runs: [ + coordinationCheck({ + id: 7999, + name: "E2E / PR Gate Coordination", + }), + ], + }, + ], + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain("E2E / PR Gate: final evidence changed"); + }); + + it.each([ + { + condition: "fails", + status: "COMPLETED", + conclusion: "FAILURE", + resultField: "failingChecks", + expected: "Repository policy: FAILURE", + }, + { + condition: "remains pending", + status: "IN_PROGRESS", + conclusion: undefined, + resultField: "pendingChecks", + expected: "Repository policy", + }, + ])("rejects a non-required repository check that $condition after final E2E evidence", ({ + status, + conclusion, + resultField, + expected, + }) => { + const result = runGateWithCoordinator({ + additionalChecksAfterE2eEvidence: [ + { + __typename: "CheckRun", + name: "Repository policy", + detailsUrl: "https://github.com/NVIDIA/NemoClaw/runs/99001", + startedAt: "2026-01-01T00:02:40Z", + completedAt: status === "COMPLETED" ? "2026-01-01T00:02:50Z" : undefined, + status, + conclusion, + }, + ], + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci[resultField]).toContain(expected); + }); + + it("rejects a changed passing required-check record during final observation", () => { + const result = runGateWithCoordinator({ + finalCheckOverrides: { + checks: { startedAt: "2026-01-01T00:00:01Z" }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "Required check rollup changed during gate evaluation", + ); + }); + + it("rejects a required check that fails during final observation", () => { + const result = runGateWithCoordinator({ + finalCheckOverrides: { + checks: { conclusion: "FAILURE" }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain("checks: FAILURE"); + }); + + it("rejects a required check that becomes pending during final observation", () => { + const result = runGateWithCoordinator({ + finalCheckOverrides: { + checks: { conclusion: undefined, status: "IN_PROGRESS" }, + }, + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.pendingChecks).toContain("checks"); + }); + + it("rejects a partition capped before its complete workflow-run inventory", () => { + const cappedIds = Array.from({ length: 1_000 }, (_value, index) => 30_000 + index); + const result = runGateWithCoordinator({ + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: paginatedRunIds(cappedIds), + totalCount: 1_001, + fallbackCreatedAt: "2025-12-31T06:00:00Z", + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500]], + }, + ], + }); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); + + const rejectedCoordinatorEvidence: Array< + CoordinatorFixture & { + condition: string; + } + > = [ + { + condition: "an inclusive boundary repeats changed metadata for the same run", + coordinator: { createdAt: "2026-01-01T00:00:00Z" }, + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: [[9500]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500]], + runOverrides: { + "9500": { updatedAt: "2026-01-01T00:02:33Z" }, + }, + }, + ], + }, + { + condition: "the same run appears outside adjacent partition boundaries", + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: [[10000]], + fallbackCreatedAt: "2025-12-31T06:00:00Z", + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[10000, 9500]], + fallbackCreatedAt: "2026-01-01T00:01:00Z", + }, + ], + }, + { + condition: "coordinator metadata changes between inventory and detail reads", + coordinatorList: { updatedAt: "2026-01-01T00:02:33Z" }, + }, + { + condition: "a partition reports inconsistent totals across pages", + coordinatorRunPartitions: [ + { + createdRange: DEFAULT_COORDINATOR_RANGES[0], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[1], + runPages: [[]], + }, + { + createdRange: DEFAULT_COORDINATOR_RANGES[2], + runPages: [[9500], []], + pageTotalCounts: [1, 2], + }, + ], + }, + { + condition: "the custom check has no coordinator", + coordinatorRunPages: [[]], + }, + { + condition: "two coordinator runs enclose the current custom check", + coordinatorRunPages: [[9500, 9501]], + extraRuns: { "9501": e2eCoordinatorRun() }, + }, + { + condition: "older same-revision coordinator metadata is malformed", + coordinatorRunPages: [[9501, 9500]], + extraRuns: { + "9501": { + ...historicalCoordinatorRun(), + repository: "example/NemoClaw", + }, + }, + }, + { + condition: "an older same-revision coordinator run failed", + coordinatorRunPages: [[9501, 9500]], + extraRuns: { + "9501": { + ...historicalCoordinatorRun(), + conclusion: "failure", + }, + }, + }, + { + condition: "an older same-revision coordinate job failed", + coordinatorRunPages: [[9501, 9500]], + extraRuns: { + "9501": { + ...historicalCoordinatorRun(), + jobs: coordinatorJobs({ + conclusion: "failure", + startedAt: "2026-01-01T00:00:20Z", + completedAt: "2026-01-01T00:00:59Z", + }), + }, + }, + }, + { + condition: "an older same-revision coordinate job completes after its run", + coordinatorRunPages: [[9501, 9500]], + extraRuns: { + "9501": { + ...historicalCoordinatorRun(), + jobs: coordinatorJobs({ + startedAt: "2026-01-01T00:00:20Z", + completedAt: "2026-01-01T00:01:01Z", + }), + }, + }, + }, + { + condition: "a failed same-revision coordinator overlaps the current custom check", + coordinatorRunPages: [[9501, 9500]], + extraRuns: { + "9501": e2eCoordinatorRun("failure", coordinatorJobs({ conclusion: "failure" })), + }, + }, + { + condition: "the coordinator title does not match", + coordinator: { displayTitle: "E2E Gate coordinate from another workflow" }, + }, + { + condition: "the coordinator title names another PR head", + coordinator: { + displayTitle: + "E2E Gate coordinate from CI PR #42 head " + + "c".repeat(40) + + " base " + + BASE_SHA + + " gate true", + }, + }, + { + condition: "the coordinator title names another PR base", + coordinator: { + displayTitle: + "E2E Gate coordinate from CI PR #42 head " + + HEAD_SHA + + " base " + + "c".repeat(40) + + " gate true", + }, + }, + { + condition: "the coordinator title sets gate false", + coordinator: { + displayTitle: + "E2E Gate coordinate from CI PR #42 head " + + HEAD_SHA + + " base " + + BASE_SHA + + " gate false", + }, + }, + { + condition: "the coordinator uses another event", + coordinator: { event: "pull_request_target" }, + }, + { + condition: "the coordinator uses another workflow path", + coordinator: { path: ".github/workflows/pr.yaml" }, + }, + { + condition: "the coordinator belongs to another repository", + coordinator: { repository: "example/NemoClaw" }, + }, + { + condition: "the coordinator head belongs to another repository", + coordinator: { headRepository: "example/NemoClaw" }, + }, + { + condition: "the coordinator uses another attempt", + coordinator: { attempt: 2 }, + }, + { + condition: "the coordinator run is not completed", + coordinator: { status: "in_progress", conclusion: null }, + }, + { + condition: "the coordinator run has a non-success conclusion", + coordinator: { conclusion: "neutral" }, + }, + { + condition: "the coordinator run failed", + coordinator: { conclusion: "failure" }, + }, + { + condition: "the coordinator has no coordinate job", + coordinatorJobs: coordinatorJobs().filter((job) => job.name !== "coordinate"), + }, + { + condition: "the coordinator has duplicate coordinate jobs", + coordinatorJobs: [ + ...coordinatorJobs(), + { + ...coordinatorJobs().find((job) => job.name === "coordinate")!, + id: 954, + }, + ], + }, + { + condition: "the coordinator job has another name", + coordinatorJobs: coordinatorJobs({ name: "finish" }), + }, + { + condition: "the coordinate job is not completed", + coordinatorJobs: coordinatorJobs({ status: "in_progress", conclusion: null }), + }, + { + condition: "the coordinate job failed", + coordinatorJobs: coordinatorJobs({ conclusion: "failure" }), + }, + { + condition: "the coordinate job has no started_at timestamp", + coordinatorJobs: coordinatorJobs({ omitStartedAt: true }), + }, + { + condition: "the coordinate job has no completed_at timestamp", + coordinatorJobs: coordinatorJobs({ omitCompletedAt: true }), + }, + { + condition: "the coordinate job started_at timestamp is null", + coordinatorJobs: coordinatorJobs({ startedAt: null }), + }, + { + condition: "the coordinate job completed_at timestamp is null", + coordinatorJobs: coordinatorJobs({ completedAt: null }), + }, + { + condition: "the coordinate job started_at timestamp is malformed", + coordinatorJobs: coordinatorJobs({ startedAt: "not-a-time" }), + }, + { + condition: "the coordinate job completed_at timestamp is malformed", + coordinatorJobs: coordinatorJobs({ completedAt: "not-a-time" }), + }, + { + condition: "the coordinate job starts after it completes", + coordinatorJobs: coordinatorJobs({ + startedAt: "2026-01-01T00:02:32Z", + completedAt: "2026-01-01T00:02:31Z", + }), + }, + { + condition: "the coordinate job completes before custom check completion", + coordinatorJobs: coordinatorJobs({ + completedAt: "2026-01-01T00:02:29Z", + }), + }, + { + condition: "the coordinate job starts after custom check completion", + coordinatorJobs: coordinatorJobs({ + startedAt: "2026-01-01T00:02:31Z", + }), + }, + { + condition: "the coordinator run has no created_at timestamp", + coordinator: { omitCreatedAt: true }, + }, + { + condition: "the coordinator run has no updated_at timestamp", + coordinator: { omitUpdatedAt: true }, + }, + { + condition: "the coordinator run created_at timestamp is malformed", + coordinator: { createdAt: "not-a-time" }, + }, + { + condition: "the coordinator run updated_at timestamp is malformed", + coordinator: { updatedAt: "not-a-time" }, + }, + { + condition: "the coordinator run is updated before it is created", + coordinator: { + createdAt: "2026-01-01T00:02:33Z", + updatedAt: "2026-01-01T00:02:32Z", + }, + }, + ]; + + it.each(rejectedCoordinatorEvidence)("rejects terminal custom check evidence when $condition", ({ + condition: _condition, + ...fixture + }) => { + const result = runGateWithCoordinator(fixture); + const output = JSON.parse(result.stdout); + + expect(output).toMatchObject({ + allPass: false, + gates: { ci: { pass: false } }, + }); + expect(output.gates.ci.failingChecks).toContain( + "E2E / PR Gate: latest attempt evidence incomplete", + ); + }); +}); diff --git a/test/skills/check-gates-evidence.test.ts b/test/skills/check-gates-evidence.test.ts index 2e8e2d13f16..2a1c3029946 100644 --- a/test/skills/check-gates-evidence.test.ts +++ b/test/skills/check-gates-evidence.test.ts @@ -905,7 +905,15 @@ describe("maintainer merge-gate contributor compliance", () => { ], { "400": { - ...exactDiffGateRun("success", [{ id: 40, name: "E2E / PR Gate" }]), + ...exactDiffGateRun("success", [ + { id: 40, name: "E2E / PR Gate" }, + { + id: 42, + name: "initialize", + startedAt: "2026-01-01T00:01:00Z", + completedAt: "2026-01-01T00:03:00Z", + }, + ]), pullRequests: [], createdAt: "2026-01-01T00:01:00Z", updatedAt: "2026-01-01T00:03:00Z", diff --git a/test/skills/check-gates-test-fixtures.ts b/test/skills/check-gates-test-fixtures.ts index e7f9a3cc495..2df5a281526 100644 --- a/test/skills/check-gates-test-fixtures.ts +++ b/test/skills/check-gates-test-fixtures.ts @@ -50,6 +50,10 @@ interface ActionJobFixture { name: string; status?: string; conclusion?: string | null; + startedAt?: string | null; + completedAt?: string | null; + omitStartedAt?: boolean; + omitCompletedAt?: boolean; } interface ActionRunFixture { @@ -62,11 +66,14 @@ interface ActionRunFixture { nextConclusion?: string | null; jobs?: ActionJobFixture[]; jobPages?: ActionJobFixture[][]; - createdAt?: string; - updatedAt?: string; + createdAt?: string | null; + updatedAt?: string | null; + omitCreatedAt?: boolean; + omitUpdatedAt?: boolean; headSha?: string; headBranch?: string; headRepository?: string; + repository?: string; pullRequestHeadSha?: string; pullRequests?: unknown[]; baseSha?: string; @@ -77,6 +84,20 @@ interface ActionRunFixture { conclusion?: string | null; } +interface CoordinatorRunPartitionFixture { + createdRange: string; + runPages: number[][]; + event?: "workflow_run" | "workflow_dispatch"; + totalCount?: number; + pageTotalCounts?: number[]; + fallbackCreatedAt?: string; + runOverrides?: Record>; + finalRunPages?: number[][]; + finalTotalCount?: number; + finalPageTotalCounts?: number[]; + finalRunOverrides?: Record>; +} + interface ComplianceFixture { body: string; checkConclusions?: Record; @@ -123,11 +144,18 @@ interface ComplianceFixture { verified: boolean; reason?: string; actionRunAttempts?: Record; + coordinatorListAttempts?: Record; + coordinatorRunPages?: number[][]; + coordinatorRunPartitions?: CoordinatorRunPartitionFixture[]; issueEventPages?: unknown[]; coordinationCheckPages?: unknown[]; formerCoordinationCheckPages?: unknown[]; + finalCoordinationCheckPages?: unknown[]; + finalFormerCoordinationCheckPages?: unknown[]; + observationTime?: string; finalPr?: Record; finalPrAfterCurrentBase?: Record; + finalPrAfterCiEvidence?: Record; } interface ComparatorFixture extends ComplianceFixture { @@ -147,7 +175,7 @@ function successfulRequiredChecksWithoutE2e() { } function successfulRequiredChecks() { - return REQUIRED_CHECK_NAMES.map((name) => requiredCheck(name)); + return [...REQUIRED_CHECK_NAMES.map((name) => requiredCheck(name)), initialE2eSeedCheck()]; } function requiredCheck(name: string, conclusion = "SUCCESS") { @@ -165,6 +193,30 @@ function requiredCheck(name: string, conclusion = "SUCCESS") { return e2eGateCheck([runId, jobId, conclusion, undefined, undefined, workflowName, name]); } +function initialE2eSeedJobs(): ActionJobFixture[] { + return [ + { id: 471, name: "cancel-superseded" }, + { id: 472, name: "initialize" }, + { id: 473, name: "coordinate", conclusion: "skipped" }, + ].map((job) => ({ + ...job, + startedAt: "2026-01-01T00:01:00Z", + completedAt: "2026-01-01T00:01:31Z", + })); +} + +function initialE2eSeedCheck() { + return { + __typename: "CheckRun", + name: "initialize", + workflowName: "E2E / PR Gate Controller", + detailsUrl: "https://github.com/NVIDIA/NemoClaw/actions/runs/407/job/472", + startedAt: "2026-01-01T00:01:00Z", + status: "COMPLETED", + conclusion: "SUCCESS", + }; +} + function e2eGateCheck(check: E2eCheckFixture, index = 0) { const [runId, jobId, conclusion, startedAt, detailsUrl, workflowName, name] = check; return { @@ -201,6 +253,60 @@ function exactDiffGateRun(result: string, jobs: ActionJobFixture[], attempt = 1) }; } +function e2eCoordinatorRun( + result = "success", + jobs: ActionJobFixture[] = [ + { + id: 951, + name: "coordinate", + startedAt: "2026-01-01T00:02:00Z", + completedAt: "2026-01-01T00:02:31Z", + }, + { + id: 952, + name: "initialize", + conclusion: "skipped", + startedAt: "2026-01-01T00:02:00Z", + completedAt: "2026-01-01T00:02:00Z", + }, + { + id: 953, + name: "cancel-superseded", + conclusion: "skipped", + startedAt: "2026-01-01T00:02:00Z", + completedAt: "2026-01-01T00:02:00Z", + }, + ], +): ActionRunFixture { + return { + attempt: 1, + createdAt: "2026-01-01T00:02:00Z", + updatedAt: "2026-01-01T00:02:32Z", + headSha: BASE_SHA, + headBranch: "main", + headRepository: "NVIDIA/NemoClaw", + repository: "NVIDIA/NemoClaw", + pullRequests: [], + displayTitle: `E2E Gate coordinate from CI PR #42 head ${HEAD_SHA} base ${BASE_SHA} gate true`, + event: "workflow_run", + path: ".github/workflows/pr-e2e-gate.yaml", + status: "completed", + conclusion: result, + jobs, + }; +} + +function e2eManualCoordinatorRun( + result = "success", + jobs: ActionJobFixture[] = e2eCoordinatorRun().jobs ?? [], +): ActionRunFixture { + return { + ...e2eCoordinatorRun(result, jobs), + displayTitle: `E2E Gate approve PR #42 head ${HEAD_SHA} base ${BASE_SHA}`, + event: "workflow_dispatch", + }; +} + function installerHashRun( result: string, jobs: ActionJobFixture[], @@ -267,20 +373,29 @@ function runGate(fixture: ComplianceFixture) { const bin = path.join(tmp, "bin"); fs.mkdirSync(bin); const ghPath = path.join(bin, "gh"); + const clockPath = path.join(tmp, "clock.mjs"); + const observationTime = + fixture.observationTime ?? + fixture.coordinatorRunPartitions?.at(-1)?.createdRange.split("..")[1] ?? + "2026-01-01T00:03:00Z"; + fs.writeFileSync(clockPath, `Date.now = () => ${Date.parse(observationTime)};\n`); const headRepository = fixture.headRepository ?? "NVIDIA/NemoClaw"; const [headRepositoryOwner, headRepositoryName] = headRepository.split("/"); + const defaultStatusChecks = + fixture.checkNames === undefined + ? successfulRequiredChecks().map((check) => { + const conclusion = fixture.checkConclusions?.[check.name]; + return conclusion === undefined ? check : { ...check, conclusion }; + }) + : fixture.checkNames.map((name) => requiredCheck(name, fixture.checkConclusions?.[name])); const pr = { number: 42, title: "fix(policy): align maintainer workflow", url: "https://github.com/NVIDIA/NemoClaw/pull/42", body: fixture.body, files: [], - statusCheckRollup: - fixture.statusChecks ?? - (fixture.checkNames ?? REQUIRED_CHECK_NAMES).map((name) => - requiredCheck(name, fixture.checkConclusions?.[name]), - ), + statusCheckRollup: fixture.statusChecks ?? defaultStatusChecks, mergeable: fixture.mergeable ?? "MERGEABLE", mergeStateStatus: fixture.mergeStateStatus ?? "CLEAN", state: "OPEN", @@ -300,6 +415,10 @@ function runGate(fixture: ComplianceFixture) { }; const finalPr = { ...pr, ...fixture.finalPr }; const finalPrAfterCurrentBase = { ...finalPr, ...fixture.finalPrAfterCurrentBase }; + const finalPrAfterCiEvidence = { + ...finalPrAfterCurrentBase, + ...fixture.finalPrAfterCiEvidence, + }; const contributorCommitPages = ( fixture.contributorCommitPages ?? [ [ @@ -369,7 +488,10 @@ function runGate(fixture: ComplianceFixture) { const formerCoordinationCheckPages = fixture.formerCoordinationCheckPages ?? [ { total_count: 0, check_runs: [] }, ]; - const actionRunCases = Object.entries({ + const finalCoordinationCheckPages = fixture.finalCoordinationCheckPages ?? coordinationCheckPages; + const finalFormerCoordinationCheckPages = + fixture.finalFormerCoordinationCheckPages ?? formerCoordinationCheckPages; + const actionRunFixtures: Record = { "90": prWorkflowRun( "success", [ @@ -392,50 +514,78 @@ function runGate(fixture: ComplianceFixture) { path: ".github/workflows/dco-check.yaml", }, "94": exactDiffGateRun("success", [{ id: 1, name: "E2E / PR Gate" }]), + "407": { + ...exactDiffGateRun("success", initialE2eSeedJobs()), + createdAt: "2026-01-01T00:01:00Z", + updatedAt: "2026-01-01T00:01:31Z", + headRepository, + pullRequests: [], + }, + "9500": e2eCoordinatorRun(), ...fixture.actionRunAttempts, - }) + }; + const actionRunData = (runId: string, value: ActionRunFixture): Record => ({ + id: Number(runId), + run_attempt: value.attempt, + ...(value.omitCreatedAt + ? {} + : { + created_at: value.createdAt === undefined ? "2026-01-01T00:01:00Z" : value.createdAt, + }), + ...(value.omitUpdatedAt + ? {} + : { + updated_at: value.updatedAt === undefined ? "2026-01-01T00:03:00Z" : value.updatedAt, + }), + event: value.event, + path: value.path, + status: value.status, + conclusion: value.conclusion, + display_title: value.displayTitle, + repository: { full_name: value.repository ?? "NVIDIA/NemoClaw" }, + ...(value.headSha ? { head_sha: value.headSha } : {}), + ...(value.headBranch ? { head_branch: value.headBranch } : {}), + ...(value.headRepository ? { head_repository: { full_name: value.headRepository } } : {}), + ...(value.pullRequests !== undefined + ? { pull_requests: value.pullRequests } + : value.headSha + ? { + pull_requests: value.baseSha + ? [ + { + number: 42, + head: { sha: value.pullRequestHeadSha ?? value.headSha }, + base: { sha: value.baseSha }, + }, + ] + : [], + } + : {}), + }); + const actionRunCases = Object.entries(actionRunFixtures) .flatMap(([runId, value]) => { const jobPages = (value.jobPages ?? [value.jobs ?? []]).map((page) => - page.map((job) => ({ + page.map(({ startedAt, completedAt, omitStartedAt, omitCompletedAt, ...job }) => ({ ...job, status: job.status ?? "completed", conclusion: job.conclusion === undefined ? "success" : job.conclusion, + ...(omitStartedAt + ? {} + : { started_at: startedAt === undefined ? "2026-01-01T00:01:00Z" : startedAt }), + ...(omitCompletedAt + ? {} + : { + completed_at: completedAt === undefined ? "2026-01-01T00:03:00Z" : completedAt, + }), })), ); const jobs = jobPages.flat(); - const runData = { - run_attempt: value.attempt, - created_at: value.createdAt ?? "2026-01-01T00:01:00Z", - updated_at: value.updatedAt ?? "2026-01-01T00:03:00Z", - event: value.event, - path: value.path, - status: value.status, - conclusion: value.conclusion, - display_title: value.displayTitle, - ...(value.headSha ? { head_sha: value.headSha } : {}), - ...(value.headBranch ? { head_branch: value.headBranch } : {}), - ...(value.headRepository ? { head_repository: { full_name: value.headRepository } } : {}), - ...(value.pullRequests !== undefined - ? { pull_requests: value.pullRequests } - : value.headSha - ? { - pull_requests: value.baseSha - ? [ - { - number: 42, - head: { sha: value.pullRequestHeadSha ?? value.headSha }, - base: { sha: value.baseSha }, - }, - ] - : [], - } - : {}), - }; + const runData = actionRunData(runId, value); const refreshedRunData = { ...runData, run_attempt: value.nextAttempt ?? value.attempt, - created_at: value.nextCreatedAt ?? runData.created_at, - updated_at: value.nextUpdatedAt ?? runData.updated_at, + ...(value.nextCreatedAt === undefined ? {} : { created_at: value.nextCreatedAt }), + ...(value.nextUpdatedAt === undefined ? {} : { updated_at: value.nextUpdatedAt }), display_title: value.nextDisplayTitle ?? runData.display_title, status: value.nextStatus ?? runData.status, conclusion: value.nextConclusion === undefined ? runData.conclusion : value.nextConclusion, @@ -454,13 +604,106 @@ function runGate(fixture: ComplianceFixture) { ]; }) .join("\n"); + const coordinatorRunPages = fixture.coordinatorRunPages ?? [[9500]]; + const configuredCoordinatorRunPartitions = fixture.coordinatorRunPartitions ?? [ + { + createdRange: "2025-12-31T00:00:00Z..2025-12-31T12:00:00Z", + runPages: [[]], + }, + { + createdRange: "2025-12-31T12:00:00Z..2026-01-01T00:00:00Z", + runPages: [[]], + }, + { + createdRange: "2026-01-01T00:00:00Z..2026-01-01T00:03:00Z", + runPages: coordinatorRunPages, + }, + ]; + const coordinatorRunPartitions = [...configuredCoordinatorRunPartitions]; + const coordinatorCreatedRanges = [ + ...new Set(configuredCoordinatorRunPartitions.map((partition) => partition.createdRange)), + ]; + for (const event of ["workflow_run", "workflow_dispatch"] as const) { + for (const createdRange of coordinatorCreatedRanges) { + if ( + !coordinatorRunPartitions.some( + (partition) => + (partition.event ?? "workflow_run") === event && + partition.createdRange === createdRange, + ) + ) { + coordinatorRunPartitions.push({ createdRange, event, runPages: [[]] }); + } + } + } + const coordinatorWorkflowRunCases = coordinatorRunPartitions + .map((partition, partitionIndex) => { + const fallbackCreatedAt = + partition.fallbackCreatedAt ?? partition.createdRange.split("..")[0]; + const event = partition.event ?? "workflow_run"; + const responsePages = ( + runPages: number[][], + totalCount: number | undefined, + pageTotalCounts: number[] | undefined, + runOverrides: Record> | undefined, + ) => { + const partitionTotal = totalCount ?? runPages.flat().length; + return runPages.map((page, pageIndex) => ({ + total_count: pageTotalCounts?.[pageIndex] ?? partitionTotal, + workflow_runs: page.map((runId) => { + const listedValue = + fixture.coordinatorListAttempts?.[String(runId)] ?? actionRunFixtures[String(runId)]; + const value = + listedValue && runOverrides?.[String(runId)] + ? { ...listedValue, ...runOverrides[String(runId)] } + : listedValue; + return value + ? actionRunData(String(runId), value) + : { + id: runId, + created_at: fallbackCreatedAt, + display_title: `unrelated workflow run ${runId}`, + event, + }; + }), + })); + }; + const pages = responsePages( + partition.runPages, + partition.totalCount, + partition.pageTotalCounts, + partition.runOverrides, + ); + const finalPages = responsePages( + partition.finalRunPages ?? partition.runPages, + partition.finalTotalCount ?? partition.totalCount, + partition.finalPageTotalCounts ?? partition.pageTotalCounts, + partition.finalRunOverrides ?? partition.runOverrides, + ); + const query = + "api --paginate --slurp repos/NVIDIA/NemoClaw/actions/workflows/pr-e2e-gate.yaml/runs?event=" + + event + + "&created=" + + encodeURIComponent(partition.createdRange) + + "&per_page=100"; + const marker = path.join(tmp, `coordinator-partition-${partitionIndex}-seen`); + return ` ${shellSingleQuote(query)}) if mkdir ${shellSingleQuote(marker)} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(pages))}; else printf '%s' ${shellSingleQuote(JSON.stringify(finalPages))}; fi ;;`; + }) + .join("\n"); + const coordinationCheckMarker = path.join(tmp, "coordination-checks-seen"); + const formerCoordinationCheckMarker = path.join(tmp, "former-coordination-checks-seen"); + const finalPrReadMarker = path.join(tmp, "final-pr-read"); fs.writeFileSync( ghPath, `#!/usr/bin/env bash set -euo pipefail +if [ -d ${shellSingleQuote(finalPrReadMarker)} ]; then + echo "unexpected gh args after final PR read: $*" >&2 + exit 9 +fi case "$*" in - "pr view"*) if mkdir ${shellSingleQuote(path.join(tmp, "pr-view-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(pr))}; elif [ -d ${shellSingleQuote(path.join(tmp, "current-base-seen"))} ]; then printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCurrentBase))}; else printf '%s' ${shellSingleQuote(JSON.stringify(finalPr))}; fi ;; + "pr view"*) if mkdir ${shellSingleQuote(path.join(tmp, "pr-view-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(pr))}; elif mkdir ${shellSingleQuote(path.join(tmp, "pr-before-final-ci-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCurrentBase))}; else mkdir -p ${shellSingleQuote(finalPrReadMarker)}; printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCiEvidence))}; fi ;; *"ContributorCommits"*) printf '%s' ${shellSingleQuote(contributorCommitOutput)} ;; *"ContributorReviews"*) printf '%s' ${shellSingleQuote(contributorReviewOutput)} ;; *"CurrentBaseRef"*) mkdir -p ${shellSingleQuote(path.join(tmp, "current-base-seen"))}; printf '%s' ${shellSingleQuote(currentBaseOutput)} ;; @@ -468,8 +711,9 @@ case "$*" in "api repos/NVIDIA/NemoClaw/issues/42/comments"*) printf '%s' '{"id":1,"body":"ordinary comment","user":{"login":"reviewer"},"updated_at":"2026-01-01T00:00:00Z"}' ;; "api repos/NVIDIA/NemoClaw/pulls/42/commits"*) printf '%s' ${shellSingleQuote(commitOutput)} ;; "api --paginate --slurp repos/NVIDIA/NemoClaw/issues/42/events?per_page=100") printf '%s' ${shellSingleQuote(JSON.stringify(issueEventPages))} ;; - "api --paginate --slurp repos/NVIDIA/NemoClaw/commits/${HEAD_SHA}/check-runs?check_name=E2E%20%2F%20PR%20Gate&filter=all&per_page=100") printf '%s' ${shellSingleQuote(JSON.stringify(coordinationCheckPages))} ;; - "api --paginate --slurp repos/NVIDIA/NemoClaw/commits/${HEAD_SHA}/check-runs?check_name=E2E%20%2F%20PR%20Gate%20Coordination&filter=all&per_page=100") printf '%s' ${shellSingleQuote(JSON.stringify(formerCoordinationCheckPages))} ;; + "api --paginate --slurp repos/NVIDIA/NemoClaw/commits/${HEAD_SHA}/check-runs?check_name=E2E%20%2F%20PR%20Gate&filter=all&per_page=100") if mkdir ${shellSingleQuote(coordinationCheckMarker)} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(coordinationCheckPages))}; else printf '%s' ${shellSingleQuote(JSON.stringify(finalCoordinationCheckPages))}; fi ;; + "api --paginate --slurp repos/NVIDIA/NemoClaw/commits/${HEAD_SHA}/check-runs?check_name=E2E%20%2F%20PR%20Gate%20Coordination&filter=all&per_page=100") if mkdir ${shellSingleQuote(formerCoordinationCheckMarker)} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(formerCoordinationCheckPages))}; else printf '%s' ${shellSingleQuote(JSON.stringify(finalFormerCoordinationCheckPages))}; fi ;; +${coordinatorWorkflowRunCases} ${actionRunCases} *) echo "unexpected gh args: $*" >&2; exit 9 ;; esac @@ -481,6 +725,8 @@ esac return spawnSync( process.execPath, [ + "--import", + clockPath, "--experimental-strip-types", "--no-warnings", ".agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts", @@ -563,6 +809,7 @@ export type { ActionRunFixture, ComparatorFixture, ComplianceFixture, + CoordinatorRunPartitionFixture, E2eCheckFixture, }; export { @@ -572,6 +819,8 @@ export { E2E_COORDINATION_EXTERNAL_ID, E2E_COORDINATION_NAME, e2eChecks, + e2eCoordinatorRun, + e2eManualCoordinatorRun, e2eGateCheck, e2eJobs, e2eRunFixture, From a925d49bdef56c3fd5b596fc86fb7cf7a9c135ef Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 12:34:36 -0700 Subject: [PATCH 02/11] fix(maintainer): type coordinator evidence fixtures Signed-off-by: Carlos Villela --- test/skills/check-gates-coordinator-evidence.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index 34cca51ec17..cb4c37c59d8 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -281,15 +281,17 @@ function runGateWithCoordinator({ const jobs = seedRun.jobs ?? seedJobs(); const statusChecks = [ ...successfulRequiredChecks().filter((check) => check.name !== "initialize"), - ...(includeInitialSeedEvidence ? seedStatusChecks(407, jobs, seedRun.createdAt) : []), + ...(includeInitialSeedEvidence + ? seedStatusChecks(407, jobs, seedRun.createdAt ?? undefined) + : []), ...additionalStatusChecks, ]; const finalStatusChecks = Object.keys(finalCheckOverrides).length > 0 - ? statusChecks.map((check) => ({ - ...check, - ...finalCheckOverrides[check.name], - })) + ? statusChecks.map((check) => { + const overrides = check.name === undefined ? undefined : finalCheckOverrides[check.name]; + return { ...check, ...overrides }; + }) : undefined; const coordinatorJobs = configuredJobs ?? coordinator.jobs ?? defaultCoordinator.jobs; return runGate({ From 246d47c3a335354638abeabf7ab509f251f639db Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 13:52:41 -0700 Subject: [PATCH 03/11] test(maintainer): align E2E gate evidence fixtures Signed-off-by: Carlos Villela --- .../check-gates-coordinator-evidence.test.ts | 9 +++ test/skills/check-gates-evidence.test.ts | 69 ++++++++++--------- test/skills/check-gates-fork-evidence.test.ts | 5 +- test/skills/check-gates-retry-history.test.ts | 25 ++++++- 4 files changed, 72 insertions(+), 36 deletions(-) diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index cb4c37c59d8..f0c9299b090 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -698,6 +698,15 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { startedAt: "2026-01-01T23:59:55Z", completedAt: "2026-01-02T00:00:44Z", }), + seedRun: { + createdAt: "2026-01-01T23:59:59Z", + updatedAt: "2026-01-02T00:00:11Z", + jobs: seedJobs().map((job) => ({ + ...job, + startedAt: "2026-01-02T00:00:00Z", + completedAt: "2026-01-02T00:00:11Z", + })), + }, coordinatorRunPartitions: [ { createdRange: "2026-01-01T00:00:00Z..2026-01-01T12:00:00Z", diff --git a/test/skills/check-gates-evidence.test.ts b/test/skills/check-gates-evidence.test.ts index 2a1c3029946..24037ee2483 100644 --- a/test/skills/check-gates-evidence.test.ts +++ b/test/skills/check-gates-evidence.test.ts @@ -542,7 +542,10 @@ describe("maintainer merge-gate contributor compliance", () => { expect(output.gates.ci).toMatchObject({ pass: false, - failingChecks: ["E2E / PR Gate: latest attempt evidence incomplete"], + failingChecks: [ + "E2E / PR Gate: latest attempt evidence incomplete", + "initialize: latest attempt evidence incomplete", + ], }); }); @@ -897,39 +900,41 @@ describe("maintainer merge-gate contributor compliance", () => { }); }); it("uses an envelope-bound E2E run when a later association-less label run is skipped", () => { - const result = runGate( - e2eRunFixture( - [ - [400, 40, "SUCCESS"], - [401, 41, "SKIPPED"], - ], - { - "400": { - ...exactDiffGateRun("success", [ - { id: 40, name: "E2E / PR Gate" }, - { - id: 42, - name: "initialize", - startedAt: "2026-01-01T00:01:00Z", - completedAt: "2026-01-01T00:03:00Z", - }, - ]), - pullRequests: [], - createdAt: "2026-01-01T00:01:00Z", - updatedAt: "2026-01-01T00:03:00Z", - }, - "401": { - ...exactDiffGateRun("skipped", [ - { id: 41, name: "E2E / PR Gate", conclusion: "skipped" }, - ]), - pullRequests: [], - createdAt: "2026-01-01T00:04:00Z", - updatedAt: "2026-01-01T00:05:00Z", - displayTitle: `E2E Gate PR #42 head ${HEAD_SHA} base ${BASE_SHA} gate false`, - }, + const fixture = e2eRunFixture( + [ + [400, 40, "SUCCESS"], + [401, 41, "SKIPPED"], + ], + { + "400": { + ...exactDiffGateRun("success", [ + { id: 40, name: "E2E / PR Gate" }, + { + id: 42, + name: "initialize", + startedAt: "2026-01-01T00:01:00Z", + completedAt: "2026-01-01T00:03:00Z", + }, + ]), + pullRequests: [], + createdAt: "2026-01-01T00:01:00Z", + updatedAt: "2026-01-01T00:03:00Z", }, - ), + "401": { + ...exactDiffGateRun("skipped", [ + { id: 41, name: "E2E / PR Gate", conclusion: "skipped" }, + ]), + pullRequests: [], + createdAt: "2026-01-01T00:04:00Z", + updatedAt: "2026-01-01T00:05:00Z", + displayTitle: `E2E Gate PR #42 head ${HEAD_SHA} base ${BASE_SHA} gate false`, + }, + }, ); + const result = runGate({ + ...fixture, + statusChecks: fixture.statusChecks?.filter((check) => check.name !== "initialize"), + }); expect(JSON.parse(result.stdout)).toMatchObject({ allPass: true, diff --git a/test/skills/check-gates-fork-evidence.test.ts b/test/skills/check-gates-fork-evidence.test.ts index 4c65a859aa7..38c38892054 100644 --- a/test/skills/check-gates-fork-evidence.test.ts +++ b/test/skills/check-gates-fork-evidence.test.ts @@ -196,7 +196,10 @@ describe("maintainer merge-gate fork evidence", () => { expect(output.gates.ci).toMatchObject({ pass: false, - failingChecks: ["E2E / PR Gate: latest attempt evidence incomplete"], + failingChecks: [ + "E2E / PR Gate: latest attempt evidence incomplete", + "initialize: latest attempt evidence incomplete", + ], }); }); }); diff --git a/test/skills/check-gates-retry-history.test.ts b/test/skills/check-gates-retry-history.test.ts index 53ace525fd7..f318b7702e5 100644 --- a/test/skills/check-gates-retry-history.test.ts +++ b/test/skills/check-gates-retry-history.test.ts @@ -22,7 +22,21 @@ function retryableFailure(id: number, reason: string, title = "Retryable E2E fai } function gateOutput(checkRuns: unknown[]) { - const currentCheckId = Math.max(...checkRuns.map((check) => (check as { id: number }).id)); + const orderedCheckIds = checkRuns + .map((check) => (check as { id: number }).id) + .sort((left, right) => left - right); + const checkRunsWithTiming = checkRuns.map((check) => { + const record = check as Record & { id: number }; + const position = orderedCheckIds.indexOf(record.id); + return { + ...record, + started_at: + position === 0 + ? "2026-01-01T00:01:30Z" + : `2026-01-01T00:02:${String(position).padStart(2, "0")}Z`, + }; + }); + const currentCheckId = orderedCheckIds.at(-1)!; return JSON.parse( runGate({ body: SIGNED_BODY, @@ -35,7 +49,9 @@ function gateOutput(checkRuns: unknown[]) { } : check, ), - coordinationCheckPages: [{ total_count: checkRuns.length, check_runs: checkRuns }], + coordinationCheckPages: [ + { total_count: checkRunsWithTiming.length, check_runs: checkRunsWithTiming }, + ], }).stdout, ); } @@ -43,7 +59,10 @@ function gateOutput(checkRuns: unknown[]) { function expectIncompleteEvidence(checkRuns: unknown[]) { expect(gateOutput(checkRuns).gates.ci).toMatchObject({ pass: false, - failingChecks: ["E2E / PR Gate: latest attempt evidence incomplete"], + failingChecks: [ + "E2E / PR Gate: latest attempt evidence incomplete", + "initialize: latest attempt evidence incomplete", + ], }); } From 7ab7f9132db06ec6837f5930303abdf5887bd41c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 14:08:41 -0700 Subject: [PATCH 04/11] fix(maintainer): reject overlapping E2E retry lineage Signed-off-by: Carlos Villela --- .../scripts/check-gates.ts | 2 +- .../check-gates-coordinator-evidence.test.ts | 29 +++++++++++++++++++ test/skills/check-gates-retry-history.test.ts | 7 +++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index 66cba78f0db..39b85ed5809 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -1074,7 +1074,7 @@ function selectedE2eLineageTiming( !Number.isFinite(startedAt) || !Number.isFinite(completedAt) || startedAt > completedAt || - (timing.at(-1)?.startedAt ?? Number.NEGATIVE_INFINITY) > startedAt + (timing.at(-1)?.completedAt ?? Number.NEGATIVE_INFINITY) > startedAt ) { return null; } diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index f0c9299b090..6be60137490 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -578,6 +578,35 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { }); }); + it("rejects a retry check that starts before the previous check completes", () => { + const fixture = delayedRetryFixture(); + const result = runGateWithCoordinator({ + ...fixture, + coordinationCheckPages: [ + { + total_count: 2, + check_runs: [ + retryableFailure(7999, "2026-08-04T14:07:50Z", "2026-08-04T14:28:07Z"), + coordinationCheck(fixture.customCheck), + ], + }, + ], + }); + + expect(JSON.parse(result.stdout)).toMatchObject({ + allPass: false, + gates: { + ci: { + pass: false, + failingChecks: [ + "E2E / PR Gate: latest attempt evidence incomplete", + "initialize: latest attempt evidence incomplete", + ], + }, + }, + }); + }); + it("rejects a stale seed with no predecessor for a newer coordination check", () => { const result = runGateWithCoordinator(delayedRetryFixture()); diff --git a/test/skills/check-gates-retry-history.test.ts b/test/skills/check-gates-retry-history.test.ts index f318b7702e5..201c08c0540 100644 --- a/test/skills/check-gates-retry-history.test.ts +++ b/test/skills/check-gates-retry-history.test.ts @@ -25,6 +25,7 @@ function gateOutput(checkRuns: unknown[]) { const orderedCheckIds = checkRuns .map((check) => (check as { id: number }).id) .sort((left, right) => left - right); + const lastPosition = orderedCheckIds.length - 1; const checkRunsWithTiming = checkRuns.map((check) => { const record = check as Record & { id: number }; const position = orderedCheckIds.indexOf(record.id); @@ -34,6 +35,12 @@ function gateOutput(checkRuns: unknown[]) { position === 0 ? "2026-01-01T00:01:30Z" : `2026-01-01T00:02:${String(position).padStart(2, "0")}Z`, + completed_at: + position === lastPosition + ? "2026-01-01T00:02:30Z" + : position === 0 + ? "2026-01-01T00:01:31Z" + : `2026-01-01T00:02:${String(position + 1).padStart(2, "0")}Z`, }; }); const currentCheckId = orderedCheckIds.at(-1)!; From 55dbfc2803315cc5989d9cfa2bb3dd64e223fe09 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 14:49:22 -0700 Subject: [PATCH 05/11] test(maintainer): cover rejected retry contexts Signed-off-by: Carlos Villela --- test/skills/check-gates-coordinator-evidence.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index 6be60137490..5f44ce7efa2 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -600,6 +600,8 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { pass: false, failingChecks: [ "E2E / PR Gate: latest attempt evidence incomplete", + "cancel-superseded: latest attempt evidence incomplete", + "coordinate: latest attempt evidence incomplete", "initialize: latest attempt evidence incomplete", ], }, From ad934030928ce09d9e96a51e783c6ed05d31baad Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 15:18:35 -0700 Subject: [PATCH 06/11] fix(maintainer): retain CI across canceled metadata edits Signed-off-by: Carlos Villela --- .agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts | 2 +- test/skills/check-gates-evidence.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index 39b85ed5809..f11d51d8809 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -1589,7 +1589,7 @@ function currentCheckRollup( run.event !== "pull_request" || run.path !== ".github/workflows/pr.yaml" || run.status !== "COMPLETED" || - run.conclusion !== "SUCCESS" + (run.conclusion !== "SUCCESS" && run.conclusion !== "CANCELLED") ) { return "not_metadata_edit"; } diff --git a/test/skills/check-gates-evidence.test.ts b/test/skills/check-gates-evidence.test.ts index 24037ee2483..aa2e4f3b79b 100644 --- a/test/skills/check-gates-evidence.test.ts +++ b/test/skills/check-gates-evidence.test.ts @@ -1037,7 +1037,7 @@ describe("maintainer merge-gate contributor compliance", () => { failingChecks: INCOMPLETE_E2E, }); }); - it("keeps substantive PR CI ahead of a later metadata-only edit run", () => { + it("keeps substantive PR CI ahead of a later canceled metadata-only edit run", () => { const checkRun = ( name: string, runId: number, @@ -1074,7 +1074,7 @@ describe("maintainer merge-gate contributor compliance", () => { true, ), "801": prWorkflowRun( - "success", + "cancelled", prWorkflowJobs("skipped", { checks: { conclusion: "success" }, changes: { conclusion: "skipped" }, From 447497866fdf92ffabb5688bb7ae644ce26809c0 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 16:53:22 -0700 Subject: [PATCH 07/11] fix(maintainer): harden merge-gate evidence Signed-off-by: Carlos Villela --- .../scripts/check-gates.ts | 186 ++++++++++++++++-- .../nemoclaw-maintainer-day/scripts/shared.ts | 1 + test/skills/check-gates-compliance.test.ts | 26 +++ .../check-gates-coordinator-evidence.test.ts | 30 ++- test/skills/check-gates-evidence.test.ts | 23 ++- test/skills/check-gates-retry-history.test.ts | 24 +-- test/skills/check-gates-test-fixtures.ts | 52 ++++- 7 files changed, 304 insertions(+), 38 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index f11d51d8809..f2c9d94d0cf 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -409,11 +409,26 @@ const E2E_COORDINATOR_MAX_PARTITIONS = Math.ceil( E2E_COORDINATOR_INVENTORY_MAX_MS / E2E_COORDINATOR_PARTITION_MS, ); const GITHUB_WORKFLOW_RUN_RESULT_CAP = 1000; +const E2E_COORDINATOR_RUN_PAGE_PROJECTION = + "{total_count,workflow_runs:[.workflow_runs[]|{id,run_attempt,event,display_title,path," + + "head_branch,head_sha,status,conclusion,repository:{full_name:.repository.full_name}," + + "head_repository:{full_name:.head_repository.full_name},created_at,updated_at}]}"; function formatGitHubTimestamp(value: number): string { return new Date(value).toISOString().replace(".000Z", "Z"); } +function ghJsonPages(args: string[]): unknown[] | null { + const output = run("gh", args); + if (!output) return null; + try { + return output.split("\n").map((page) => JSON.parse(page)); + } catch { + process.stderr.write(`[check-gates] gh JSON page parse failed for: gh ${args.join(" ")}\n`); + return null; + } +} + function e2eCoordinatorRunPartitions( historyStartedAt: number, observationAt: number, @@ -482,6 +497,7 @@ function parseE2eCoordinatorRun( value: unknown, repo: string, exactDiff: ExactDiffIdentity, + trustedWorkflowSha: string | null, ): E2eCoordinatorRunMetadata | null { if (typeof value !== "object" || value === null || Array.isArray(value)) return null; const record = value as Record; @@ -511,7 +527,8 @@ function parseE2eCoordinatorRun( record.path !== E2E_COORDINATOR_WORKFLOW_PATH || record.head_branch !== "main" || !/^[0-9a-f]{40}$/u.test(headSha) || - (event === "workflow_run" && headSha !== exactDiff.baseSha) || + trustedWorkflowSha === null || + headSha !== trustedWorkflowSha || !status || !ACTION_STATUSES.has(status) || (conclusion !== null && @@ -576,14 +593,15 @@ function evaluateE2eCoordinatorCandidate( listedValue: unknown, repo: string, exactDiff: ExactDiffIdentity, + trustedWorkflowSha: string | null, coordinationStartedAt: number, coordinationCompletedAt: number, ): E2eCoordinatorCandidateEvaluation { - const listedRun = parseE2eCoordinatorRun(listedValue, repo, exactDiff); + const listedRun = parseE2eCoordinatorRun(listedValue, repo, exactDiff, trustedWorkflowSha); if (!listedRun) return { result: false }; const firstRunResponse = ghJson(["api", "repos/" + repo + "/actions/runs/" + listedRun.id]); if (firstRunResponse === null) return { result: null }; - const firstRun = parseE2eCoordinatorRun(firstRunResponse, repo, exactDiff); + const firstRun = parseE2eCoordinatorRun(firstRunResponse, repo, exactDiff, trustedWorkflowSha); if (!firstRun || !sameE2eCoordinatorRun(listedRun, firstRun)) return { result: false }; const jobPages = ghJson([ @@ -640,7 +658,12 @@ function evaluateE2eCoordinatorCandidate( const refreshedRunResponse = ghJson(["api", "repos/" + repo + "/actions/runs/" + firstRun.id]); if (refreshedRunResponse === null) return { result: null }; - const refreshedRun = parseE2eCoordinatorRun(refreshedRunResponse, repo, exactDiff); + const refreshedRun = parseE2eCoordinatorRun( + refreshedRunResponse, + repo, + exactDiff, + trustedWorkflowSha, + ); if (!refreshedRun || !sameE2eCoordinatorRun(firstRun, refreshedRun)) { return { result: false }; } @@ -736,6 +759,7 @@ function evaluateE2eCoordinatorCandidate( function fetchE2eCoordinatorEvidence( repo: string, exactDiff: ExactDiffIdentity, + trustedWorkflowSha: string | null, historyStartedAt: number, coordinationStartedAt: number, coordinationCompletedAt: number, @@ -752,10 +776,11 @@ function fetchE2eCoordinatorEvidence( ".." + formatGitHubTimestamp(partition.completedAt), ); - const pages = ghJson([ + const pages = ghJsonPages([ "api", "--paginate", - "--slurp", + "--jq", + E2E_COORDINATOR_RUN_PAGE_PROJECTION, "repos/" + repo + "/actions/workflows/pr-e2e-gate.yaml/runs?event=" + @@ -861,6 +886,7 @@ function fetchE2eCoordinatorEvidence( candidate, repo, exactDiff, + trustedWorkflowSha, coordinationStartedAt, coordinationCompletedAt, ); @@ -1113,6 +1139,7 @@ function e2eCoordinationHistoryStartedAt( function fetchE2eCoordinationEvidence( repo: string, exactDiff: ExactDiffIdentity, + trustedWorkflowSha: string | null, ): E2eCoordinationEvidence { const checkSnapshot = fetchE2eCoordinationCheckSnapshot(repo, exactDiff); if (!checkSnapshot) return { valid: null }; @@ -1143,6 +1170,7 @@ function fetchE2eCoordinationEvidence( ? fetchE2eCoordinatorEvidence( repo, exactDiff, + trustedWorkflowSha, historyStartedAt, startedAt, completedAt, @@ -2246,10 +2274,11 @@ function checkCi( statusCheckRollup: StatusCheck[] | null, repo: string, exactDiff: ExactDiffIdentity, + trustedWorkflowSha: string | null, ): CiEvaluation { const e2eCoordinationEvidence = statusCheckRollup && statusCheckRollup.length > 0 - ? fetchE2eCoordinationEvidence(repo, exactDiff) + ? fetchE2eCoordinationEvidence(repo, exactDiff, trustedWorkflowSha) : { valid: false }; return { gate: evaluateCiRollup(statusCheckRollup, repo, exactDiff, e2eCoordinationEvidence), @@ -2719,7 +2748,7 @@ function checkContributorCompliance( // Main // --------------------------------------------------------------------------- -interface PrRevisionSnapshot { +interface PrRevisionIdentity { title: string; body: string; state: string; @@ -2731,6 +2760,9 @@ interface PrRevisionSnapshot { headRefName: string; baseRefName: string; headRepository: string; +} + +interface PrRevisionSnapshot extends PrRevisionIdentity { statusCheckRollup: StatusCheck[]; } @@ -2813,9 +2845,98 @@ function fetchPrRevisionSnapshot(repo: string, number: number): PrRevisionSnapsh }; } +function fetchFinalPrIdentitySnapshot( + repo: string, + number: number, +): { revision: PrRevisionIdentity; currentBaseSha: string } | null { + const [owner, name, extra] = repo.split("/"); + if (!owner || !name || extra) return null; + + const response = ghJson([ + "api", + "graphql", + "-F", + `owner=${owner}`, + "-F", + `name=${name}`, + "-F", + `number=${number}`, + "-f", + `query=query FinalPrIdentity($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + title + body + state + isDraft + mergeable + mergeStateStatus + headRefOid + baseRefOid + headRefName + baseRefName + headRepository { name nameWithOwner } + headRepositoryOwner { login } + baseRef { target { oid } } + } + } + }`, + ]) as { + data?: { repository?: { pullRequest?: Record } }; + } | null; + const record = response?.data?.repository?.pullRequest; + if (!record) return null; + + const headRepository = parseHeadRepository(record.headRepository, record.headRepositoryOwner); + const baseRef = + typeof record.baseRef === "object" && record.baseRef !== null && !Array.isArray(record.baseRef) + ? (record.baseRef as Record) + : null; + const target = + typeof baseRef?.target === "object" && baseRef.target !== null && !Array.isArray(baseRef.target) + ? (baseRef.target as Record) + : null; + const currentBaseSha = target?.oid; + if ( + typeof record.title !== "string" || + typeof record.body !== "string" || + typeof record.state !== "string" || + typeof record.isDraft !== "boolean" || + typeof record.mergeable !== "string" || + typeof record.mergeStateStatus !== "string" || + typeof record.headRefOid !== "string" || + !/^[0-9a-f]{40}$/iu.test(record.headRefOid) || + typeof record.baseRefOid !== "string" || + !/^[0-9a-f]{40}$/iu.test(record.baseRefOid) || + typeof record.headRefName !== "string" || + typeof record.baseRefName !== "string" || + typeof currentBaseSha !== "string" || + !/^[0-9a-f]{40}$/iu.test(currentBaseSha) || + !headRepository + ) { + return null; + } + return { + revision: { + title: record.title, + body: record.body, + state: record.state, + isDraft: record.isDraft, + mergeable: record.mergeable, + mergeStateStatus: record.mergeStateStatus, + headRefOid: record.headRefOid, + baseRefOid: record.baseRefOid, + headRefName: record.headRefName, + baseRefName: record.baseRefName, + headRepository, + }, + currentBaseSha, + }; +} + function checkFinalRevision( - captured: PrRevisionSnapshot, - current: PrRevisionSnapshot | null, + captured: PrRevisionIdentity, + current: PrRevisionIdentity | null, currentBaseSha: string | null, ): ReturnType { if (!current) { @@ -2924,7 +3045,8 @@ function main(): void { headRefName: prData.headRefName, headRepository, }; - const initialCi = checkCi(prData.statusCheckRollup, repo, exactDiff); + const currentBaseSha = fetchCurrentBaseSha(repo, prNumber); + const initialCi = checkCi(prData.statusCheckRollup, repo, exactDiff, currentBaseSha); const coderabbit = checkCodeRabbit(repo, prNumber); const riskyCodeTested = checkRiskyCodeTested(prData.files ?? []); const contributorCompliance = checkContributorCompliance( @@ -2952,7 +3074,6 @@ function main(): void { headRepository, statusCheckRollup: prData.statusCheckRollup, }; - const currentBaseSha = fetchCurrentBaseSha(repo, prNumber); const revisionBeforeFinalCi = fetchPrRevisionSnapshot(repo, prNumber); const finalCiActionEvidence = createCiActionEvidenceCache(); const evaluatedRollupCi = checkFinalCi( @@ -2963,7 +3084,7 @@ function main(): void { finalCiActionEvidence, ); const finalE2eEvidence = evaluatedRollupCi.pass - ? fetchE2eCoordinationEvidence(repo, exactDiff) + ? fetchE2eCoordinationEvidence(repo, exactDiff, currentBaseSha) : { valid: false }; const evaluatedCi = checkFinalE2eEvidence( evaluatedRollupCi, @@ -2981,14 +3102,47 @@ function main(): void { finalE2eEvidence, finalCiActionEvidence, ); + const finalIdentity = fetchFinalPrIdentitySnapshot(repo, prNumber); + const finalRevision = finalIdentity?.revision ?? null; + const finalCurrentBaseSha = finalIdentity?.currentBaseSha ?? null; + const stableCurrentBaseSha = + currentBaseSha && finalCurrentBaseSha && currentBaseSha === finalCurrentBaseSha + ? finalCurrentBaseSha + : null; + const baseRevisionGate: ReturnType = + stableCurrentBaseSha !== null + ? { + pass: true, + details: "Base branch revision remained stable during gate evaluation", + mergeable: capturedRevision.mergeable, + mergeStateStatus: capturedRevision.mergeStateStatus, + baseSha: capturedRevision.baseRefOid, + currentBaseSha: stableCurrentBaseSha, + } + : { + pass: false, + details: + currentBaseSha && finalCurrentBaseSha + ? "Base branch revision changed during gate evaluation; rerun the gate checker" + : "Unable to verify the current base branch revision", + mergeable: capturedRevision.mergeable, + mergeStateStatus: capturedRevision.mergeStateStatus, + baseSha: capturedRevision.baseRefOid, + ...(finalCurrentBaseSha ? { currentBaseSha: finalCurrentBaseSha } : {}), + }; const revisionBeforeFinalCiGate = checkFinalRevision( capturedRevision, revisionBeforeFinalCi, - currentBaseSha, + stableCurrentBaseSha, ); - const conflicts = revisionBeforeFinalCiGate.pass - ? checkFinalRevision(revisionBeforeFinalCi!, currentRevision, currentBaseSha) + const revisionAfterFinalEvidenceGate = revisionBeforeFinalCiGate.pass + ? checkFinalRevision(revisionBeforeFinalCi!, currentRevision, stableCurrentBaseSha) : revisionBeforeFinalCiGate; + const conflicts = !baseRevisionGate.pass + ? baseRevisionGate + : revisionAfterFinalEvidenceGate.pass + ? checkFinalRevision(currentRevision!, finalRevision, stableCurrentBaseSha) + : revisionAfterFinalEvidenceGate; const output: GateOutput = { pr: prNumber, diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts index 85e87942493..fa39cfb6769 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts @@ -24,6 +24,7 @@ export const RISKY_PATTERNS: RegExp[] = [ /^nemoclaw\/src\/blueprint\//, /^nemoclaw-blueprint\//, /^\.github\/workflows\//, + /^\.agents\/skills\/nemoclaw-maintainer-day\/scripts\/check-gates\.ts$/, /\.prek\./, /policy/i, /ssrf/i, diff --git a/test/skills/check-gates-compliance.test.ts b/test/skills/check-gates-compliance.test.ts index e21378a1df2..3cba8957573 100644 --- a/test/skills/check-gates-compliance.test.ts +++ b/test/skills/check-gates-compliance.test.ts @@ -6,6 +6,32 @@ import { describe, expect, it } from "vitest"; import { REQUIRED_CHECK_NAMES, runComparatorGate, runGate } from "./check-gates-test-fixtures.ts"; describe("maintainer merge-gate contributor compliance", () => { + it("classifies the merge-gate checker as risky code with direct test coverage", () => { + const output = JSON.parse( + runGate({ + body: "Signed-off-by: Example User ", + verified: true, + files: [ + { + path: ".agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts", + status: "modified", + }, + { + path: "test/skills/check-gates-compliance.test.ts", + status: "modified", + }, + ], + }).stdout, + ); + + expect(output.gates.riskyCodeTested).toMatchObject({ + pass: true, + details: "1 risky file(s) changed; test files present in PR", + riskyFiles: [".agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts"], + hasTests: true, + }); + }); + it("passes when the PR body has DCO and every commit is GitHub Verified", () => { const result = runGate({ body: "## Summary\n\nPolicy alignment.\n\nSigned-off-by: Example User ", diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index 5f44ce7efa2..97d35f8bb94 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -109,6 +109,7 @@ interface CoordinatorFixture { includeInitialSeedEvidence?: boolean; observationTime?: string; seedRun?: Partial; + currentBaseSha?: string | null; } const LIVE_FORK_COORDINATOR_RANGES = [ @@ -134,7 +135,6 @@ function authorizedForkLifecycleFixture( ...e2eManualCoordinatorRun(), createdAt: "2026-08-04T14:38:13Z", updatedAt: "2026-08-04T14:57:43Z", - headSha: "d".repeat(40), jobs: coordinatorJobs({ startedAt: "2026-08-04T14:38:14Z", completedAt: "2026-08-04T14:57:42Z", @@ -276,6 +276,7 @@ function runGateWithCoordinator({ includeInitialSeedEvidence = true, observationTime, seedRun = {}, + currentBaseSha, }: CoordinatorFixture = {}) { const defaultCoordinator = e2eCoordinatorRun(); const jobs = seedRun.jobs ?? seedJobs(); @@ -299,6 +300,7 @@ function runGateWithCoordinator({ verified: true, headRepository, observationTime, + currentBaseSha, statusChecks, coordinatorRunPages, coordinatorRunPartitions, @@ -359,11 +361,25 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { }); }); + it("authenticates coordinator runs against the captured workflow revision", () => { + const workflowSha = "d".repeat(40); + const result = runGateWithCoordinator({ + coordinator: { headSha: workflowSha }, + currentBaseSha: workflowSha, + }); + const output = JSON.parse(result.stdout); + + expect(output.gates.ci).toMatchObject({ pass: true }); + expect(output.gates.conflicts).toMatchObject({ + pass: false, + baseSha: BASE_SHA, + currentBaseSha: workflowSha, + }); + expect(output.allPass).toBe(false); + }); + it("rejects a manual coordinator for a fork with no automatic predecessor", () => { - const manualCoordinator = { - ...e2eManualCoordinatorRun(), - headSha: "d".repeat(40), - }; + const manualCoordinator = e2eManualCoordinatorRun(); const result = runGateWithCoordinator({ headRepository: "example/fork", coordinator: manualCoordinator, @@ -1336,6 +1352,10 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { condition: "the coordinator uses another workflow path", coordinator: { path: ".github/workflows/pr.yaml" }, }, + { + condition: "the coordinator uses another trusted workflow revision", + coordinator: { headSha: "d".repeat(40) }, + }, { condition: "the coordinator belongs to another repository", coordinator: { repository: "example/NemoClaw" }, diff --git a/test/skills/check-gates-evidence.test.ts b/test/skills/check-gates-evidence.test.ts index aa2e4f3b79b..698038cd866 100644 --- a/test/skills/check-gates-evidence.test.ts +++ b/test/skills/check-gates-evidence.test.ts @@ -80,6 +80,25 @@ describe("maintainer merge-gate contributor compliance", () => { expect(output.allPass).toBe(false); }); + it("fails closed when the base branch changes during gate evaluation", () => { + const finalCurrentBaseSha = "c".repeat(40); + const output = JSON.parse( + runGate({ + body: "Signed-off-by: Example User ", + verified: true, + finalCurrentBaseSha, + }).stdout, + ); + + expect(output.gates.conflicts).toMatchObject({ + pass: false, + details: "Base branch revision changed during gate evaluation; rerun the gate checker", + baseSha: BASE_SHA, + currentBaseSha: finalCurrentBaseSha, + }); + expect(output.allPass).toBe(false); + }); + it("fails closed while GitHub has not determined mergeability", () => { const result = runGate({ body: "Signed-off-by: Example User ", @@ -139,12 +158,12 @@ describe("maintainer merge-gate contributor compliance", () => { expect(output.allPass).toBe(false); }); - it("makes the PR revision snapshot the final remote read", () => { + it("makes the final PR revision snapshot the final remote read", () => { const output = JSON.parse( runGate({ body: "Signed-off-by: Example User ", verified: true, - finalPrAfterCurrentBase: { headRefOid: "c".repeat(40) }, + finalPrAfterFinalCi: { headRefOid: "c".repeat(40) }, }).stdout, ); diff --git a/test/skills/check-gates-retry-history.test.ts b/test/skills/check-gates-retry-history.test.ts index 201c08c0540..f674d068c58 100644 --- a/test/skills/check-gates-retry-history.test.ts +++ b/test/skills/check-gates-retry-history.test.ts @@ -25,22 +25,15 @@ function gateOutput(checkRuns: unknown[]) { const orderedCheckIds = checkRuns .map((check) => (check as { id: number }).id) .sort((left, right) => left - right); - const lastPosition = orderedCheckIds.length - 1; + const timingWindowStart = Date.parse("2026-01-01T00:01:30Z"); + const slotDuration = 60_000 / orderedCheckIds.length; const checkRunsWithTiming = checkRuns.map((check) => { const record = check as Record & { id: number }; const position = orderedCheckIds.indexOf(record.id); return { ...record, - started_at: - position === 0 - ? "2026-01-01T00:01:30Z" - : `2026-01-01T00:02:${String(position).padStart(2, "0")}Z`, - completed_at: - position === lastPosition - ? "2026-01-01T00:02:30Z" - : position === 0 - ? "2026-01-01T00:01:31Z" - : `2026-01-01T00:02:${String(position + 1).padStart(2, "0")}Z`, + started_at: new Date(timingWindowStart + position * slotDuration).toISOString(), + completed_at: new Date(timingWindowStart + (position + 1) * slotDuration).toISOString(), }; }); const currentCheckId = orderedCheckIds.at(-1)!; @@ -84,6 +77,15 @@ describe("maintainer merge-gate E2E retry history", () => { expect(output).toMatchObject({ allPass: true, gates: { ci: { pass: true } } }); }); + it("accepts retry history with more than 31 completed checks", () => { + const retryHistory = Array.from({ length: 39 }, (_value, index) => + retryableFailure(8001 + index, "prerequisite-ci"), + ); + const output = gateOutput([coordinationCheck({ id: 8040 }), ...retryHistory]); + + expect(output).toMatchObject({ allPass: true, gates: { ci: { pass: true } } }); + }); + it.each([ ["an older success", [coordinationCheck({ id: 8002 }), coordinationCheck({ id: 8001 })]], [ diff --git a/test/skills/check-gates-test-fixtures.ts b/test/skills/check-gates-test-fixtures.ts index 2df5a281526..b8fe93cba9d 100644 --- a/test/skills/check-gates-test-fixtures.ts +++ b/test/skills/check-gates-test-fixtures.ts @@ -141,6 +141,8 @@ interface ComplianceFixture { mergeable?: string; mergeStateStatus?: string; currentBaseSha?: string | null; + finalCurrentBaseSha?: string | null; + files?: Array<{ path: string; status: string }>; verified: boolean; reason?: string; actionRunAttempts?: Record; @@ -156,6 +158,7 @@ interface ComplianceFixture { finalPr?: Record; finalPrAfterCurrentBase?: Record; finalPrAfterCiEvidence?: Record; + finalPrAfterFinalCi?: Record; } interface ComparatorFixture extends ComplianceFixture { @@ -394,7 +397,7 @@ function runGate(fixture: ComplianceFixture) { title: "fix(policy): align maintainer workflow", url: "https://github.com/NVIDIA/NemoClaw/pull/42", body: fixture.body, - files: [], + files: fixture.files ?? [], statusCheckRollup: fixture.statusChecks ?? defaultStatusChecks, mergeable: fixture.mergeable ?? "MERGEABLE", mergeStateStatus: fixture.mergeStateStatus ?? "CLEAN", @@ -419,6 +422,10 @@ function runGate(fixture: ComplianceFixture) { ...finalPrAfterCurrentBase, ...fixture.finalPrAfterCiEvidence, }; + const finalPrAfterFinalCi = { + ...finalPrAfterCiEvidence, + ...fixture.finalPrAfterFinalCi, + }; const contributorCommitPages = ( fixture.contributorCommitPages ?? [ [ @@ -478,6 +485,34 @@ function runGate(fixture: ComplianceFixture) { }, }, }); + const finalCurrentBaseSha = + fixture.finalCurrentBaseSha === undefined + ? fixture.currentBaseSha + : fixture.finalCurrentBaseSha; + const finalPrIdentityOutput = JSON.stringify({ + data: { + repository: { + pullRequest: { + title: finalPrAfterFinalCi.title, + body: finalPrAfterFinalCi.body, + state: finalPrAfterFinalCi.state, + isDraft: finalPrAfterFinalCi.isDraft, + mergeable: finalPrAfterFinalCi.mergeable, + mergeStateStatus: finalPrAfterFinalCi.mergeStateStatus, + headRefOid: finalPrAfterFinalCi.headRefOid, + baseRefOid: finalPrAfterFinalCi.baseRefOid, + headRefName: finalPrAfterFinalCi.headRefName, + baseRefName: finalPrAfterFinalCi.baseRefName, + headRepository: finalPrAfterFinalCi.headRepository, + headRepositoryOwner: finalPrAfterFinalCi.headRepositoryOwner, + baseRef: + finalCurrentBaseSha === null + ? null + : { target: { oid: finalCurrentBaseSha ?? BASE_SHA } }, + }, + }, + }, + }); const issueEventPages = fixture.issueEventPages ?? [[]]; const coordinationCheckPages = fixture.coordinationCheckPages ?? [ { @@ -680,14 +715,22 @@ function runGate(fixture: ComplianceFixture) { partition.finalPageTotalCounts ?? partition.pageTotalCounts, partition.finalRunOverrides ?? partition.runOverrides, ); + const projection = + "{total_count,workflow_runs:[.workflow_runs[]|{id,run_attempt,event,display_title,path," + + "head_branch,head_sha,status,conclusion,repository:{full_name:.repository.full_name}," + + "head_repository:{full_name:.head_repository.full_name},created_at,updated_at}]}"; const query = - "api --paginate --slurp repos/NVIDIA/NemoClaw/actions/workflows/pr-e2e-gate.yaml/runs?event=" + + "api --paginate --jq " + + projection + + " repos/NVIDIA/NemoClaw/actions/workflows/pr-e2e-gate.yaml/runs?event=" + event + "&created=" + encodeURIComponent(partition.createdRange) + "&per_page=100"; const marker = path.join(tmp, `coordinator-partition-${partitionIndex}-seen`); - return ` ${shellSingleQuote(query)}) if mkdir ${shellSingleQuote(marker)} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(pages))}; else printf '%s' ${shellSingleQuote(JSON.stringify(finalPages))}; fi ;;`; + const output = pages.map((page) => JSON.stringify(page)).join("\n"); + const finalOutput = finalPages.map((page) => JSON.stringify(page)).join("\n"); + return ` ${shellSingleQuote(query)}) if mkdir ${shellSingleQuote(marker)} 2>/dev/null; then printf '%s' ${shellSingleQuote(output)}; else printf '%s' ${shellSingleQuote(finalOutput)}; fi ;;`; }) .join("\n"); const coordinationCheckMarker = path.join(tmp, "coordination-checks-seen"); @@ -703,10 +746,11 @@ if [ -d ${shellSingleQuote(finalPrReadMarker)} ]; then exit 9 fi case "$*" in - "pr view"*) if mkdir ${shellSingleQuote(path.join(tmp, "pr-view-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(pr))}; elif mkdir ${shellSingleQuote(path.join(tmp, "pr-before-final-ci-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCurrentBase))}; else mkdir -p ${shellSingleQuote(finalPrReadMarker)}; printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCiEvidence))}; fi ;; + "pr view"*) if mkdir ${shellSingleQuote(path.join(tmp, "pr-view-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(pr))}; elif mkdir ${shellSingleQuote(path.join(tmp, "pr-before-final-ci-seen"))} 2>/dev/null; then printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCurrentBase))}; else printf '%s' ${shellSingleQuote(JSON.stringify(finalPrAfterCiEvidence))}; fi ;; *"ContributorCommits"*) printf '%s' ${shellSingleQuote(contributorCommitOutput)} ;; *"ContributorReviews"*) printf '%s' ${shellSingleQuote(contributorReviewOutput)} ;; *"CurrentBaseRef"*) mkdir -p ${shellSingleQuote(path.join(tmp, "current-base-seen"))}; printf '%s' ${shellSingleQuote(currentBaseOutput)} ;; + *"FinalPrIdentity"*) mkdir -p ${shellSingleQuote(finalPrReadMarker)}; printf '%s' ${shellSingleQuote(finalPrIdentityOutput)} ;; "api graphql"*) printf '%s' '{"data":{"repository":{"pullRequest":{"reviewThreads":{"nodes":[]}}}}}' ;; "api repos/NVIDIA/NemoClaw/issues/42/comments"*) printf '%s' '{"id":1,"body":"ordinary comment","user":{"login":"reviewer"},"updated_at":"2026-01-01T00:00:00Z"}' ;; "api repos/NVIDIA/NemoClaw/pulls/42/commits"*) printf '%s' ${shellSingleQuote(commitOutput)} ;; From 8418d609ae24bd5c2b10c82e6d4bcd2d3ed8f51b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 17:34:59 -0700 Subject: [PATCH 08/11] fix(maintainer): verify final gate snapshot Signed-off-by: Carlos Villela (cherry picked from commit 5c3e773e5f6e5b51bb0474f09d6107a08005666b) --- .../scripts/check-gates.ts | 186 +++++++++++++++++- .../check-gates-coordinator-evidence.test.ts | 8 + .../skills/check-gates-final-snapshot.test.ts | 46 +++++ test/skills/check-gates-retry-history.test.ts | 6 +- test/skills/check-gates-test-fixtures.ts | 31 ++- 5 files changed, 265 insertions(+), 12 deletions(-) create mode 100644 test/skills/check-gates-final-snapshot.test.ts diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index f2c9d94d0cf..18f11e6775c 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -2845,10 +2845,10 @@ function fetchPrRevisionSnapshot(repo: string, number: number): PrRevisionSnapsh }; } -function fetchFinalPrIdentitySnapshot( +function fetchFinalPrSnapshot( repo: string, number: number, -): { revision: PrRevisionIdentity; currentBaseSha: string } | null { +): { revision: PrRevisionSnapshot; currentBaseSha: string } | null { const [owner, name, extra] = repo.split("/"); if (!owner || !name || extra) return null; @@ -2862,7 +2862,7 @@ function fetchFinalPrIdentitySnapshot( "-F", `number=${number}`, "-f", - `query=query FinalPrIdentity($owner: String!, $name: String!, $number: Int!) { + `query=query FinalPrSnapshot($owner: String!, $name: String!, $number: Int!) { repository(owner: $owner, name: $name) { pullRequest(number: $number) { title @@ -2878,6 +2878,37 @@ function fetchFinalPrIdentitySnapshot( headRepository { name nameWithOwner } headRepositoryOwner { login } baseRef { target { oid } } + commits(last: 1) { + totalCount + nodes { + commit { + oid + statusCheckRollup { + contexts(first: 100) { + totalCount + pageInfo { hasNextPage } + nodes { + __typename + ... on CheckRun { + name + status + conclusion + startedAt + completedAt + detailsUrl + checkSuite { workflowRun { workflow { name } } } + } + ... on StatusContext { + context + state + startedAt: createdAt + } + } + } + } + } + } + } } } }`, @@ -2916,6 +2947,8 @@ function fetchFinalPrIdentitySnapshot( ) { return null; } + const statusCheckRollup = parseFinalStatusCheckRollup(record.commits, record.headRefOid); + if (!statusCheckRollup) return null; return { revision: { title: record.title, @@ -2929,11 +2962,137 @@ function fetchFinalPrIdentitySnapshot( headRefName: record.headRefName, baseRefName: record.baseRefName, headRepository, + statusCheckRollup, }, currentBaseSha, }; } +function parseFinalStatusCheckRollup( + commitsValue: unknown, + expectedHeadSha: string, +): StatusCheck[] | null { + if (typeof commitsValue !== "object" || commitsValue === null || Array.isArray(commitsValue)) { + return null; + } + const commits = commitsValue as Record; + if ( + typeof commits.totalCount !== "number" || + !Number.isInteger(commits.totalCount) || + commits.totalCount < 1 || + !Array.isArray(commits.nodes) || + commits.nodes.length !== 1 + ) { + return null; + } + const commitNode = commits.nodes[0]; + if (typeof commitNode !== "object" || commitNode === null || Array.isArray(commitNode)) { + return null; + } + const commitValue = (commitNode as Record).commit; + if (typeof commitValue !== "object" || commitValue === null || Array.isArray(commitValue)) { + return null; + } + const commit = commitValue as Record; + if (commit.oid !== expectedHeadSha) return null; + const rollupValue = commit.statusCheckRollup; + if (typeof rollupValue !== "object" || rollupValue === null || Array.isArray(rollupValue)) { + return null; + } + const contextsValue = (rollupValue as Record).contexts; + if (typeof contextsValue !== "object" || contextsValue === null || Array.isArray(contextsValue)) { + return null; + } + const contexts = contextsValue as Record; + const pageInfo = contexts.pageInfo; + if ( + typeof contexts.totalCount !== "number" || + !Number.isInteger(contexts.totalCount) || + contexts.totalCount < 0 || + !Array.isArray(contexts.nodes) || + contexts.nodes.length !== contexts.totalCount || + typeof pageInfo !== "object" || + pageInfo === null || + Array.isArray(pageInfo) || + (pageInfo as Record).hasNextPage !== false + ) { + return null; + } + + const statusCheckRollup: StatusCheck[] = []; + for (const nodeValue of contexts.nodes) { + if (typeof nodeValue !== "object" || nodeValue === null || Array.isArray(nodeValue)) { + return null; + } + const node = nodeValue as Record; + if (node.__typename === "CheckRun") { + const checkSuite = + typeof node.checkSuite === "object" && + node.checkSuite !== null && + !Array.isArray(node.checkSuite) + ? (node.checkSuite as Record) + : null; + const workflowRun = + typeof checkSuite?.workflowRun === "object" && + checkSuite.workflowRun !== null && + !Array.isArray(checkSuite.workflowRun) + ? (checkSuite.workflowRun as Record) + : null; + const workflow = + typeof workflowRun?.workflow === "object" && + workflowRun.workflow !== null && + !Array.isArray(workflowRun.workflow) + ? (workflowRun.workflow as Record) + : null; + const workflowName = workflow?.name; + const scalarFields = [ + node.name, + node.status, + node.conclusion, + node.startedAt, + node.completedAt, + node.detailsUrl, + workflowName, + ]; + if ( + typeof node.name !== "string" || + typeof node.status !== "string" || + scalarFields.some( + (value) => value !== undefined && value !== null && typeof value !== "string", + ) + ) { + return null; + } + statusCheckRollup.push({ + __typename: "CheckRun", + name: node.name, + status: node.status, + ...(typeof node.conclusion === "string" ? { conclusion: node.conclusion } : {}), + ...(typeof node.startedAt === "string" ? { startedAt: node.startedAt } : {}), + ...(typeof node.completedAt === "string" ? { completedAt: node.completedAt } : {}), + ...(typeof node.detailsUrl === "string" ? { detailsUrl: node.detailsUrl } : {}), + ...(typeof workflowName === "string" ? { workflowName } : {}), + }); + continue; + } + if ( + node.__typename !== "StatusContext" || + typeof node.context !== "string" || + typeof node.state !== "string" || + typeof node.startedAt !== "string" + ) { + return null; + } + statusCheckRollup.push({ + __typename: "StatusContext", + context: node.context, + state: node.state, + startedAt: node.startedAt, + }); + } + return statusCheckRollup; +} + function checkFinalRevision( captured: PrRevisionIdentity, current: PrRevisionIdentity | null, @@ -3093,7 +3252,7 @@ function main(): void { exactDiff, ); const currentRevision = fetchPrRevisionSnapshot(repo, prNumber); - const ci = checkLastCi( + const ciBeforeFinalSnapshot = checkLastCi( evaluatedCi, initialCi, currentRevision?.statusCheckRollup ?? null, @@ -3102,13 +3261,26 @@ function main(): void { finalE2eEvidence, finalCiActionEvidence, ); - const finalIdentity = fetchFinalPrIdentitySnapshot(repo, prNumber); - const finalRevision = finalIdentity?.revision ?? null; - const finalCurrentBaseSha = finalIdentity?.currentBaseSha ?? null; + const finalSnapshot = fetchFinalPrSnapshot(repo, prNumber); + const finalRevision = finalSnapshot?.revision ?? null; + const finalCurrentBaseSha = finalSnapshot?.currentBaseSha ?? null; const stableCurrentBaseSha = currentBaseSha && finalCurrentBaseSha && currentBaseSha === finalCurrentBaseSha ? finalCurrentBaseSha : null; + const ci = !ciBeforeFinalSnapshot.pass + ? ciBeforeFinalSnapshot + : !finalRevision + ? { pass: false, details: "Unable to verify the final PR checks" } + : checkLastCi( + ciBeforeFinalSnapshot, + initialCi, + finalRevision.statusCheckRollup, + repo, + exactDiff, + finalE2eEvidence, + finalCiActionEvidence, + ); const baseRevisionGate: ReturnType = stableCurrentBaseSha !== null ? { diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index 97d35f8bb94..038887d8f47 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -1352,6 +1352,14 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { condition: "the coordinator uses another workflow path", coordinator: { path: ".github/workflows/pr.yaml" }, }, + { + condition: "the coordinator runs on another branch", + coordinator: { headBranch: "release" }, + }, + { + condition: "the coordinator head commit SHA is malformed", + coordinator: { headSha: "not-a-sha" }, + }, { condition: "the coordinator uses another trusted workflow revision", coordinator: { headSha: "d".repeat(40) }, diff --git a/test/skills/check-gates-final-snapshot.test.ts b/test/skills/check-gates-final-snapshot.test.ts new file mode 100644 index 00000000000..f11b38aaacf --- /dev/null +++ b/test/skills/check-gates-final-snapshot.test.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { runGate, successfulRequiredChecks } from "./check-gates-test-fixtures.ts"; + +describe("maintainer merge-gate final PR snapshot", () => { + it("rejects a required check that becomes pending in the final PR observation", () => { + const output = JSON.parse( + runGate({ + body: "Signed-off-by: Example User ", + verified: true, + finalPrAfterFinalCi: { + statusCheckRollup: successfulRequiredChecks().map((check) => + check.name === "checks" + ? { ...check, status: "IN_PROGRESS", conclusion: undefined } + : check, + ), + }, + }).stdout, + ); + + expect(output.gates.ci).toMatchObject({ pass: false }); + expect(output.gates.ci.pendingChecks).toContain("checks"); + expect(output.allPass).toBe(false); + }); + + it.each([ + ["the status rollup is incomplete", { finalStatusCheckHasNextPage: true }], + ["the status rollup belongs to another commit", { finalStatusCheckCommitOid: "c".repeat(40) }], + ])("fails closed when %s in the final PR observation", (_condition, finalStatusFixture) => { + const output = JSON.parse( + runGate({ + body: "Signed-off-by: Example User ", + verified: true, + ...finalStatusFixture, + }).stdout, + ); + + expect(output.gates.ci).toMatchObject({ + pass: false, + details: "Unable to verify the final PR checks", + }); + expect(output.allPass).toBe(false); + }); +}); diff --git a/test/skills/check-gates-retry-history.test.ts b/test/skills/check-gates-retry-history.test.ts index f674d068c58..20b23a4279a 100644 --- a/test/skills/check-gates-retry-history.test.ts +++ b/test/skills/check-gates-retry-history.test.ts @@ -77,11 +77,11 @@ describe("maintainer merge-gate E2E retry history", () => { expect(output).toMatchObject({ allPass: true, gates: { ci: { pass: true } } }); }); - it("accepts retry history with more than 31 completed checks", () => { - const retryHistory = Array.from({ length: 39 }, (_value, index) => + it("accepts retry history with more than 60 completed checks", () => { + const retryHistory = Array.from({ length: 61 }, (_value, index) => retryableFailure(8001 + index, "prerequisite-ci"), ); - const output = gateOutput([coordinationCheck({ id: 8040 }), ...retryHistory]); + const output = gateOutput([coordinationCheck({ id: 8062 }), ...retryHistory]); expect(output).toMatchObject({ allPass: true, gates: { ci: { pass: true } } }); }); diff --git a/test/skills/check-gates-test-fixtures.ts b/test/skills/check-gates-test-fixtures.ts index b8fe93cba9d..5b12dae4ce3 100644 --- a/test/skills/check-gates-test-fixtures.ts +++ b/test/skills/check-gates-test-fixtures.ts @@ -159,6 +159,8 @@ interface ComplianceFixture { finalPrAfterCurrentBase?: Record; finalPrAfterCiEvidence?: Record; finalPrAfterFinalCi?: Record; + finalStatusCheckCommitOid?: string; + finalStatusCheckHasNextPage?: boolean; } interface ComparatorFixture extends ComplianceFixture { @@ -489,7 +491,15 @@ function runGate(fixture: ComplianceFixture) { fixture.finalCurrentBaseSha === undefined ? fixture.currentBaseSha : fixture.finalCurrentBaseSha; - const finalPrIdentityOutput = JSON.stringify({ + const finalStatusCheckNodes = finalPrAfterFinalCi.statusCheckRollup.map( + ({ workflowName, ...check }) => ({ + ...check, + ...(workflowName + ? { checkSuite: { workflowRun: { workflow: { name: workflowName } } } } + : {}), + }), + ); + const finalPrSnapshotOutput = JSON.stringify({ data: { repository: { pullRequest: { @@ -509,6 +519,23 @@ function runGate(fixture: ComplianceFixture) { finalCurrentBaseSha === null ? null : { target: { oid: finalCurrentBaseSha ?? BASE_SHA } }, + commits: { + totalCount: 1, + nodes: [ + { + commit: { + oid: fixture.finalStatusCheckCommitOid ?? finalPrAfterFinalCi.headRefOid, + statusCheckRollup: { + contexts: { + totalCount: finalStatusCheckNodes.length, + pageInfo: { hasNextPage: fixture.finalStatusCheckHasNextPage ?? false }, + nodes: finalStatusCheckNodes, + }, + }, + }, + }, + ], + }, }, }, }, @@ -750,7 +777,7 @@ case "$*" in *"ContributorCommits"*) printf '%s' ${shellSingleQuote(contributorCommitOutput)} ;; *"ContributorReviews"*) printf '%s' ${shellSingleQuote(contributorReviewOutput)} ;; *"CurrentBaseRef"*) mkdir -p ${shellSingleQuote(path.join(tmp, "current-base-seen"))}; printf '%s' ${shellSingleQuote(currentBaseOutput)} ;; - *"FinalPrIdentity"*) mkdir -p ${shellSingleQuote(finalPrReadMarker)}; printf '%s' ${shellSingleQuote(finalPrIdentityOutput)} ;; + *"FinalPrSnapshot"*) mkdir -p ${shellSingleQuote(finalPrReadMarker)}; printf '%s' ${shellSingleQuote(finalPrSnapshotOutput)} ;; "api graphql"*) printf '%s' '{"data":{"repository":{"pullRequest":{"reviewThreads":{"nodes":[]}}}}}' ;; "api repos/NVIDIA/NemoClaw/issues/42/comments"*) printf '%s' '{"id":1,"body":"ordinary comment","user":{"login":"reviewer"},"updated_at":"2026-01-01T00:00:00Z"}' ;; "api repos/NVIDIA/NemoClaw/pulls/42/commits"*) printf '%s' ${shellSingleQuote(commitOutput)} ;; From 1ebc2d1019cfaa9cfdbd340d28e7caed824d9a4e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 17:57:11 -0700 Subject: [PATCH 09/11] test(maintainer): cover gate evidence boundaries Signed-off-by: Carlos Villela (cherry picked from commit 3f6a4b67be3bc9d083626dcb989a431ed36ee9cb) --- .../check-gates-coordinator-evidence.test.ts | 4 ++-- .../skills/check-gates-final-snapshot.test.ts | 19 +++++++++++++++++++ test/skills/check-gates-test-fixtures.ts | 7 +++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index 038887d8f47..d7cd6d78151 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -1146,14 +1146,14 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { expect(output.gates.ci.pendingChecks).toContain("checks"); }); - it("rejects a partition capped before its complete workflow-run inventory", () => { + it("rejects a workflow-run inventory at the 1,000-result cap", () => { const cappedIds = Array.from({ length: 1_000 }, (_value, index) => 30_000 + index); const result = runGateWithCoordinator({ coordinatorRunPartitions: [ { createdRange: DEFAULT_COORDINATOR_RANGES[0], runPages: paginatedRunIds(cappedIds), - totalCount: 1_001, + totalCount: 1_000, fallbackCreatedAt: "2025-12-31T06:00:00Z", }, { diff --git a/test/skills/check-gates-final-snapshot.test.ts b/test/skills/check-gates-final-snapshot.test.ts index f11b38aaacf..e67ea9d225b 100644 --- a/test/skills/check-gates-final-snapshot.test.ts +++ b/test/skills/check-gates-final-snapshot.test.ts @@ -28,6 +28,10 @@ describe("maintainer merge-gate final PR snapshot", () => { it.each([ ["the status rollup is incomplete", { finalStatusCheckHasNextPage: true }], ["the status rollup belongs to another commit", { finalStatusCheckCommitOid: "c".repeat(40) }], + [ + "the status rollup count does not match its returned contexts", + { finalStatusContextTotalCount: successfulRequiredChecks().length + 1 }, + ], ])("fails closed when %s in the final PR observation", (_condition, finalStatusFixture) => { const output = JSON.parse( runGate({ @@ -43,4 +47,19 @@ describe("maintainer merge-gate final PR snapshot", () => { }); expect(output.allPass).toBe(false); }); + + it("accepts a multi-commit PR when the final snapshot returns its last commit", () => { + const output = JSON.parse( + runGate({ + body: "Signed-off-by: Example User ", + verified: true, + finalCommitTotalCount: 2, + }).stdout, + ); + + expect(output).toMatchObject({ + allPass: true, + gates: { ci: { pass: true } }, + }); + }); }); diff --git a/test/skills/check-gates-test-fixtures.ts b/test/skills/check-gates-test-fixtures.ts index 5b12dae4ce3..e81d4c6bfde 100644 --- a/test/skills/check-gates-test-fixtures.ts +++ b/test/skills/check-gates-test-fixtures.ts @@ -159,6 +159,8 @@ interface ComplianceFixture { finalPrAfterCurrentBase?: Record; finalPrAfterCiEvidence?: Record; finalPrAfterFinalCi?: Record; + finalCommitTotalCount?: number; + finalStatusContextTotalCount?: number; finalStatusCheckCommitOid?: string; finalStatusCheckHasNextPage?: boolean; } @@ -520,14 +522,15 @@ function runGate(fixture: ComplianceFixture) { ? null : { target: { oid: finalCurrentBaseSha ?? BASE_SHA } }, commits: { - totalCount: 1, + totalCount: fixture.finalCommitTotalCount ?? 1, nodes: [ { commit: { oid: fixture.finalStatusCheckCommitOid ?? finalPrAfterFinalCi.headRefOid, statusCheckRollup: { contexts: { - totalCount: finalStatusCheckNodes.length, + totalCount: + fixture.finalStatusContextTotalCount ?? finalStatusCheckNodes.length, pageInfo: { hasNextPage: fixture.finalStatusCheckHasNextPage ?? false }, nodes: finalStatusCheckNodes, }, From 04084d0cf9f566121be46bb365a44a0a38762550 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 20:36:37 -0700 Subject: [PATCH 10/11] test(maintainer): clarify gate evidence language Signed-off-by: Carlos Villela --- .../nemoclaw-maintainer-day/scripts/check-gates.ts | 12 +++++++----- test/skills/check-gates-coordinator-evidence.test.ts | 8 ++++---- test/skills/check-gates-evidence.test.ts | 6 +++--- test/skills/check-gates-final-snapshot.test.ts | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index 18f11e6775c..0cec7758bea 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -424,7 +424,9 @@ function ghJsonPages(args: string[]): unknown[] | null { try { return output.split("\n").map((page) => JSON.parse(page)); } catch { - process.stderr.write(`[check-gates] gh JSON page parse failed for: gh ${args.join(" ")}\n`); + process.stderr.write( + `[check-gates] The gate checker could not parse a JSON page from this command: gh ${args.join(" ")}\n`, + ); return null; } } @@ -2409,7 +2411,7 @@ function checkConflicts( if (!currentBaseSha) { return { pass: false, - details: "Unable to verify the current base branch revision", + details: "The gate checker could not verify the base SHA", mergeable: conflictStatus, mergeStateStatus: status, baseSha, @@ -3285,7 +3287,7 @@ function main(): void { stableCurrentBaseSha !== null ? { pass: true, - details: "Base branch revision remained stable during gate evaluation", + details: "The base SHA did not change during gate evaluation", mergeable: capturedRevision.mergeable, mergeStateStatus: capturedRevision.mergeStateStatus, baseSha: capturedRevision.baseRefOid, @@ -3295,8 +3297,8 @@ function main(): void { pass: false, details: currentBaseSha && finalCurrentBaseSha - ? "Base branch revision changed during gate evaluation; rerun the gate checker" - : "Unable to verify the current base branch revision", + ? "The base SHA changed during gate evaluation. Rerun the gate checker." + : "The gate checker could not verify the base SHA", mergeable: capturedRevision.mergeable, mergeStateStatus: capturedRevision.mergeStateStatus, baseSha: capturedRevision.baseRefOid, diff --git a/test/skills/check-gates-coordinator-evidence.test.ts b/test/skills/check-gates-coordinator-evidence.test.ts index d7cd6d78151..a9f423455bb 100644 --- a/test/skills/check-gates-coordinator-evidence.test.ts +++ b/test/skills/check-gates-coordinator-evidence.test.ts @@ -531,7 +531,7 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { ); }); - it("accepts the exact coordinator from a later complete workflow-runs page", () => { + it("accepts a matching coordinator from a later page of a complete workflow-run inventory", () => { const unrelatedRun = { ...e2eCoordinatorRun(), displayTitle: "E2E Gate coordinate from unrelated CI", @@ -776,7 +776,7 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { }); }); - it("accepts more than 1,000 workflow runs across complete partitions", () => { + it("accepts more than 1,000 workflow runs when each partition returns its reported total", () => { const firstPartitionIds = Array.from({ length: 600 }, (_value, index) => 10_000 + index); const secondPartitionIds = Array.from({ length: 600 }, (_value, index) => 20_000 + index); const result = runGateWithCoordinator({ @@ -985,7 +985,7 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { ); }); - it("rejects an extreme custom-check timestamp span before inventory traversal", () => { + it("rejects a custom-check timestamp span beyond the 14-day inventory limit", () => { const result = runGateWithCoordinator({ customCheck: { started_at: "2000-01-01T00:00:00Z", @@ -1361,7 +1361,7 @@ describe("maintainer merge-gate E2E coordinator evidence", () => { coordinator: { headSha: "not-a-sha" }, }, { - condition: "the coordinator uses another trusted workflow revision", + condition: "the coordinator commit SHA does not match the captured base SHA", coordinator: { headSha: "d".repeat(40) }, }, { diff --git a/test/skills/check-gates-evidence.test.ts b/test/skills/check-gates-evidence.test.ts index 698038cd866..69ff9621812 100644 --- a/test/skills/check-gates-evidence.test.ts +++ b/test/skills/check-gates-evidence.test.ts @@ -92,7 +92,7 @@ describe("maintainer merge-gate contributor compliance", () => { expect(output.gates.conflicts).toMatchObject({ pass: false, - details: "Base branch revision changed during gate evaluation; rerun the gate checker", + details: "The base SHA changed during gate evaluation. Rerun the gate checker.", baseSha: BASE_SHA, currentBaseSha: finalCurrentBaseSha, }); @@ -158,7 +158,7 @@ describe("maintainer merge-gate contributor compliance", () => { expect(output.allPass).toBe(false); }); - it("makes the final PR revision snapshot the final remote read", () => { + it("performs no remote read after the final PR snapshot", () => { const output = JSON.parse( runGate({ body: "Signed-off-by: Example User ", @@ -1056,7 +1056,7 @@ describe("maintainer merge-gate contributor compliance", () => { failingChecks: INCOMPLETE_E2E, }); }); - it("keeps substantive PR CI ahead of a later canceled metadata-only edit run", () => { + it("retains failing CI from a code-changing PR run after a later metadata-only run is canceled", () => { const checkRun = ( name: string, runId: number, diff --git a/test/skills/check-gates-final-snapshot.test.ts b/test/skills/check-gates-final-snapshot.test.ts index e67ea9d225b..fff04fd17b6 100644 --- a/test/skills/check-gates-final-snapshot.test.ts +++ b/test/skills/check-gates-final-snapshot.test.ts @@ -48,7 +48,7 @@ describe("maintainer merge-gate final PR snapshot", () => { expect(output.allPass).toBe(false); }); - it("accepts a multi-commit PR when the final snapshot returns its last commit", () => { + it("accepts a multi-commit PR when the final snapshot returns the PR commit SHA", () => { const output = JSON.parse( runGate({ body: "Signed-off-by: Example User ", From ae0df72669ab686152980e1d49527c6f000ff646 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 21:39:16 -0700 Subject: [PATCH 11/11] test(maintainers): correct gate fixture timing Signed-off-by: Carlos Villela --- test/skills/check-gates-retry-history.test.ts | 2 +- test/skills/check-gates-test-fixtures.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/skills/check-gates-retry-history.test.ts b/test/skills/check-gates-retry-history.test.ts index 20b23a4279a..d873f8abaa1 100644 --- a/test/skills/check-gates-retry-history.test.ts +++ b/test/skills/check-gates-retry-history.test.ts @@ -25,7 +25,7 @@ function gateOutput(checkRuns: unknown[]) { const orderedCheckIds = checkRuns .map((check) => (check as { id: number }).id) .sort((left, right) => left - right); - const timingWindowStart = Date.parse("2026-01-01T00:01:30Z"); + const timingWindowStart = Date.parse("2026-01-01T00:01:31Z"); const slotDuration = 60_000 / orderedCheckIds.length; const checkRunsWithTiming = checkRuns.map((check) => { const record = check as Record & { id: number }; diff --git a/test/skills/check-gates-test-fixtures.ts b/test/skills/check-gates-test-fixtures.ts index e81d4c6bfde..d2c0b113b87 100644 --- a/test/skills/check-gates-test-fixtures.ts +++ b/test/skills/check-gates-test-fixtures.ts @@ -389,7 +389,7 @@ function runGate(fixture: ComplianceFixture) { const headRepository = fixture.headRepository ?? "NVIDIA/NemoClaw"; const [headRepositoryOwner, headRepositoryName] = headRepository.split("/"); - const defaultStatusChecks = + const defaultStatusChecks: NonNullable = fixture.checkNames === undefined ? successfulRequiredChecks().map((check) => { const conclusion = fixture.checkConclusions?.[check.name];