feat(operator): gate readiness on installation completion

.github/workflows/svcaplbot-run-dyff.yml

@@ -107,6 +107,22 @@ jobs:
           echo '```diff' >> "$comment_file"
           cat "$GITHUB_WORKSPACE/pr/tmp/diff-output.txt" >> "$comment_file"
           echo '```' >> "$comment_file"
+
+          # Always publish the diff to the job summary. For pull requests from a
+          # fork, secrets — and therefore BOT_TOKEN — are not available, so this
+          # is the only channel that can carry the comparison.
+          # Truncating mid-diff would leave the ```diff fence unclosed and garble the
+          # rest of the summary, so close it explicitly and say the output was cut.
+          summary_limit=900000
+          head -c "$summary_limit" "$comment_file" >> "$GITHUB_STEP_SUMMARY"
+          if [ "$(wc -c < "$comment_file")" -gt "$summary_limit" ]; then
+            printf '\n```\n\n_Output truncated at %s bytes._\n' "$summary_limit" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
           if [ "${{ github.event_name }}" = "pull_request" ]; then
+            if [ -z "$GH_TOKEN" ]; then
+              echo "::notice::No BOT_TOKEN available (pull request from a fork) — comparison published to the job summary instead of a PR comment."
+              exit 0
+            fi
             gh pr comment ${{ github.event.pull_request.number }} --body-file "$comment_file" --create-if-none --edit-last
           fi

chart/apl/templates/NOTES.txt

@@ -2,4 +2,13 @@ The App Platform Operator has been successfully deployed on the cluster.
 
 Please inspect the output of the apl-operator deployment (apl-operator/{{ include "apl-operator.fullname" . }}) for any feedback or errors.
 
+Installing the platform takes 10-15 minutes. The operator reports Ready only once
+installation has completed, so you can wait for it:
+
+  kubectl wait --for=condition=Available deployment/{{ include "apl-operator.fullname" . }} -n apl-operator --timeout=30m
+
+Progress is readable at any time from:
+
+  kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}'
+
 Also visit https://techdocs.akamai.com/app-platform/ for further instructions and reference documentation.

chart/apl/templates/deployment.yaml

@@ -1,6 +1,7 @@
 {{- $kms := .Values.kms | default dict }}
 {{- $version := .Values.otomi.version | default .Chart.AppVersion }}
 {{- $skipDeployment := .Values.installation.skipOperatorDeployment }}
+{{- $readiness := .Values.operator.readiness }}
 {{- if not $skipDeployment }}
 apiVersion: apps/v1
 kind: Deployment
@@ -10,6 +11,8 @@ metadata:
   labels: {{- include "apl-operator.labels" . | nindent 4 }}
 spec:
   replicas: 1
+  # The rollout stays Progressing until the operator is ready, which is longer than the 600s default.
+  progressDeadlineSeconds: {{ $readiness.progressDeadlineSeconds }}
   selector:
     matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }}
   strategy:
@@ -90,7 +93,7 @@ spec:
             failureThreshold: 3
           readinessProbe:
             exec:
-              command: ["/bin/sh", "-c", "pgrep -f 'apl-operator' > /dev/null"]
+              command: ["/bin/sh", "-c", "test -f /tmp/ready"]
             initialDelaySeconds: 30
             periodSeconds: 10
             timeoutSeconds: 5

chart/apl/values.yaml

@@ -129,6 +129,10 @@ operator:
   installRetries: 1000
   installMaxTimeoutMs: 10000
 
+  readiness:
+    # 30 minutes. If the operator has not installed the platform by then, something is off.
+    progressDeadlineSeconds: 1800
+
   image:
     repository: "mirror.registry.linodelke.net/docker/linode/apl-core"
 

charts/apl-operator/templates/deployment.yaml

@@ -10,6 +10,8 @@ metadata:
 {{- end }}
 spec:
   replicas: {{ .Values.scale.replicas | default 1 }}
+  # The rollout stays Progressing until the operator is ready, which is longer than the 600s default.
+  progressDeadlineSeconds: 1800
   selector:
     matchLabels:
       {{- include "apl-operator.selectorLabels" . | nindent 6 }}
@@ -73,6 +75,16 @@ spec:
             periodSeconds: 60
             failureThreshold: 3
             timeoutSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - "test -f /tmp/ready"
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
+            timeoutSeconds: 5
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

src/operator/EXECUTION_FLOW.md

@@ -473,6 +473,77 @@ Shared by both loops with trigger-specific variations:
 8. Update apply state to 'succeeded' or 'failed'
 9. Release lock (`isApplying = false`)
 
+## Readiness and Convergence Contract
+
+Bootstrap automation needs a machine-checkable answer to "has the operator finished
+its job yet?". The operator exposes it through the readiness of its own Deployment.
+
+### The gate
+
+The operator writes `/tmp/ready` (`markOperatorReady()`) at exactly one point: after
+an apply run completes successfully. That run is what creates the ArgoCD Applications,
+so past it the platform can heal itself through ArgoCD. The `readinessProbe` on the
+apl-operator Deployment tests for that file, so:
+
+```bash
+# blocks until the operator has completed an apply run
+kubectl wait --for=condition=Available deployment/apl-operator -n apl-operator --timeout=30m
+
+# same signal, via helm
+helm install apl … --wait --timeout 30m
+```
+
+This is **not** the same as "the platform is fully up". When the operator reports
+Ready, ArgoCD is still working through the Applications it was just handed. The gate
+says the operator is finished and its reconcile loop has started — from there, health
+belongs to ArgoCD.
+
+Three properties are deliberate:
+
+- **It latches, for the life of the pod.** Readiness is never cleared by a later apply.
+  The reconcile loop applies every ~5 minutes in steady state; flipping the Deployment
+  out of `Available` on each pass would make the condition useless as a gate. Per-apply
+  status is reported through the `apl-operator-state` ConfigMap instead (below). The
+  marker lives on the pod's `/tmp` emptyDir, so it survives a container restart within
+  the pod and is only cleared when the pod itself is recreated — a rescheduled or
+  rolled-out pod goes NotReady until it completes an apply of its own.
+- **It fails closed.** If the marker cannot be written, or the apply keeps failing,
+  the pod stays NotReady. The signal never claims progress that did not happen —
+  `--wait` times out loudly rather than returning early.
+- **A first install takes 10-15 minutes.** Size `--timeout` accordingly; the
+  Deployment's `progressDeadlineSeconds` is raised to 1800 so `kubectl rollout
+  status` does not report `ProgressDeadlineExceeded` on a healthy install.
+
+### Introspection
+
+For phase detail rather than a binary gate, read the ConfigMaps in the table below:
+
+```bash
+# installation phase: pending | in-progress | completed | failed (+ attempt, timestamp)
+kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}'
+
+# last apply: commitHash, status, timestamp, trigger, errorMessage
+kubectl get cm apl-operator-state -n apl-operator -o jsonpath='{.data.state}'
+```
+
+`apl-operator-state.commitHash` is the answer to "did the operator apply *my* commit
+yet?" — poll for `status: succeeded` at the revision you pushed.
+
+### What this is not
+
+The Deployment gate covers the operator's own pipeline: essential manifests, CRDs,
+`stage=prep`, `app=core`, and the creation of the ArgoCD Applications for the
+remaining apps. Whether those Applications have actually synced and gone Healthy is
+ArgoCD's business, not this gate's.
+
+An end-to-end smoke check that the platform is externally serving is
+`https://auth.<domainSuffix>/ready` (oauth2-proxy behind the ingress). It exercises
+DNS, ingress-nginx, the TLS certificate and the auth chain, which the in-cluster
+gate does not. It is complementary, not a substitute: it needs public DNS and a
+trusted certificate, it cannot tell you *which* revision of your values converged,
+and a non-200 cannot distinguish "platform not ready" from a DNS or certificate
+problem.
+
 ## Kubernetes Resources
 
 ### ConfigMaps

src/operator/apl-operator.test.ts

@@ -5,7 +5,7 @@ import { restartPlatformAuthPods } from '../common/runtime-upgrades/restart-plat
 import { AplOperations } from './apl-operations'
 import { AplOperator, AplOperatorConfig, ApplyTrigger, OAUTH2_PROXY_ARGOCD_APP_NAME } from './apl-operator'
 import { GitRepository } from './git-repository'
-import { hasPlatformAuthPodsRestarted, markPlatformAuthPodsRestarted, updateApplyState } from './k8s'
+import { markOperatorReady, hasPlatformAuthPodsRestarted, markPlatformAuthPodsRestarted, updateApplyState } from './k8s'
 
 const mockInfoFn = jest.fn()
 const mockWarnFn = jest.fn()
@@ -65,6 +65,7 @@ jest.mock('./k8s', () => ({
   appRevisionMatches: jest.fn().mockResolvedValue(true),
   hasPlatformAuthPodsRestarted: jest.fn().mockResolvedValue(true),
   markPlatformAuthPodsRestarted: jest.fn().mockResolvedValue(undefined),
+  markOperatorReady: jest.fn(),
 }))
 
 jest.mock('../common/k8s', () => ({
@@ -224,6 +225,7 @@ describe('AplOperator', () => {
         }),
       )
 
+      expect(markOperatorReady).toHaveBeenCalled()
       expect((aplOperator as any).isApplying).toBe(false)
     })
 
@@ -275,6 +277,8 @@ describe('AplOperator', () => {
         }),
       )
 
+      // A failed apply leaves the ArgoCD Applications unaccounted for — the pod must stay NotReady.
+      expect(markOperatorReady).not.toHaveBeenCalled()
       expect((aplOperator as any).isApplying).toBe(false)
 
       expect(mockErrorFn).toHaveBeenCalledWith('[poll] Apply process failed', 'Apply failed')

src/operator/apl-operator.ts

@@ -11,7 +11,7 @@ import { ensureManifestDirectories, ensureTeamGitOpsDirectories } from '../commo
 import { getDefaultValues, writeValues } from '../common/values'
 import { AplOperations } from './apl-operations'
 import { GitRepository } from './git-repository'
-import { hasPlatformAuthPodsRestarted, markPlatformAuthPodsRestarted, updateApplyState } from './k8s'
+import { hasPlatformAuthPodsRestarted, markOperatorReady, markPlatformAuthPodsRestarted, updateApplyState } from './k8s'
 import { getErrorMessage } from './utils'
 
 export interface AplOperatorConfig {
@@ -103,6 +103,11 @@ export class AplOperator {
 
       this.d.info(`[${trigger}] Apply process completed`)
 
+      // The apply run above is what creates the ArgoCD Applications, so from here on the
+      // platform can heal itself through ArgoCD. That — not the end of the helmfile install
+      // — is what the operator being 'ready' means.
+      markOperatorReady()
+
       await updateApplyState({
         commitHash,
         status: 'succeeded',

src/operator/k8s.test.ts

@@ -1,5 +1,15 @@
-import { ApplyState, hasPlatformAuthPodsRestarted, markPlatformAuthPodsRestarted, updateApplyState } from './k8s'
+import {
+  ApplyState,
+  markOperatorReady,
+  READINESS_FILE,
+  hasPlatformAuthPodsRestarted,
+  markPlatformAuthPodsRestarted,
+  updateApplyState,
+} from './k8s'
 import { CoreV1Api, ApiException } from '@kubernetes/client-node'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
 
 jest.mock('@kubernetes/client-node', () => {
   const mocks = {
@@ -38,6 +48,7 @@ jest.mock('../common/debug', () => ({
   terminal: jest.fn().mockImplementation(() => ({
     info: jest.fn(),
     error: jest.fn(),
+    warn: jest.fn(),
   })),
 }))
 
@@ -231,3 +242,44 @@ describe('markPlatformAuthPodsRestarted', () => {
     })
   })
 })
+
+describe('markOperatorReady', () => {
+  let workDir: string
+
+  beforeEach(() => {
+    workDir = mkdtempSync(join(tmpdir(), 'apl-readiness-'))
+  })
+
+  afterEach(() => {
+    rmSync(workDir, { recursive: true, force: true })
+  })
+
+  test('defaults to the path the readinessProbe checks', () => {
+    expect(READINESS_FILE).toBe('/tmp/ready')
+  })
+
+  test('writes the readiness marker with a timestamp', () => {
+    const marker = join(workDir, 'ready')
+
+    markOperatorReady(marker)
+
+    expect(existsSync(marker)).toBe(true)
+    expect(Date.parse(readFileSync(marker, 'utf8'))).not.toBeNaN()
+  })
+
+  test('is idempotent — every apply run re-marks readiness', () => {
+    const marker = join(workDir, 'ready')
+
+    markOperatorReady(marker)
+    markOperatorReady(marker)
+
+    expect(existsSync(marker)).toBe(true)
+  })
+
+  test('never throws when the marker cannot be written, leaving the pod NotReady', () => {
+    const unwritable = join(workDir, 'does', 'not', 'exist', 'ready')
+
+    expect(() => markOperatorReady(unwritable)).not.toThrow()
+    expect(existsSync(unwritable)).toBe(false)
+  })
+})

src/operator/k8s.ts

@@ -39,6 +39,25 @@ export function updateHeartbeatFile(): void {
   writeFileSync('/tmp/heartbeat', '')
 }
 
+export const READINESS_FILE = '/tmp/ready'
+
+/**
+ * Idempotent, and safe to call on every apply. Readiness latches: the marker is never
+ * cleared while a later apply runs, because the steady-state reconcile loop would
+ * otherwise flap the Deployment's Available condition. Per-apply status lives in the
+ * apl-operator-state ConfigMap.
+ */
+export function markOperatorReady(filePath: string = READINESS_FILE): void {
+  const d = terminal('operator:k8s:markOperatorReady')
+  try {
+    writeFileSync(filePath, new Date().toISOString())
+    d.info(`Wrote readiness marker ${filePath}`)
+  } catch (error) {
+    // Non-fatal: a missing marker keeps the pod NotReady, which is the safe direction.
+    d.warn(`Failed to write readiness marker ${filePath}:`, getErrorMessage(error))
+  }
+}
+
 export async function updateApplyState(
   state: ApplyState,
   namespace: string = APL_OPERATOR_NS,