From 78a39ea65e78fa24345bdffbb178f471e27be37f Mon Sep 17 00:00:00 2001 From: JSON Date: Mon, 27 Apr 2026 15:54:32 +0900 Subject: [PATCH 1/2] feat(infra): add monitoring stack e2e validation to ci Catches prometheus-operator silent reconcile failures before merge by booting a kind cluster, installing kube-prometheus-stack with prod values, and asserting that: - prometheus-operator log has no `failed to (initialize|provision|sync)` - alertmanager-generated secret reflects our config (discord_configs + inhibit_rules present) This class of failure (e.g. operator vendored alertmanager library lag on `webhook_url_file`) is invisible to helm template, kubeconform, and even amtool check-config because it depends on operator runtime reconciliation. Static analysis cannot catch it; only an actual install into a sandbox cluster can. Verified locally that this workflow catches the exact silent fail that escaped review on the previous alertmanager wiring PR. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci-infra.yml | 127 +++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/.github/workflows/ci-infra.yml b/.github/workflows/ci-infra.yml index d2a03bc845..8a38650e15 100644 --- a/.github/workflows/ci-infra.yml +++ b/.github/workflows/ci-infra.yml @@ -103,3 +103,130 @@ jobs: done echo "All keys are valid" + + monitoring-e2e: + name: Monitoring Stack E2E (kind) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Check if monitoring source code has changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + monitoring: + - 'infra/k8s/monitoring/**' + - '.github/workflows/ci-infra.yml' + + - name: Setup helm + if: steps.filter.outputs.monitoring == 'true' + uses: azure/setup-helm@v4 + with: + version: 'v3.19.2' + + - name: Boot kind cluster + if: steps.filter.outputs.monitoring == 'true' + uses: helm/kind-action@v1 + with: + cluster_name: pr-validation + version: v0.24.0 + wait: 60s + + - name: Create namespaces and dummy discord webhook secret + if: steps.filter.outputs.monitoring == 'true' + run: | + set -euo pipefail + kubectl create namespace monitoring-grafana + kubectl create namespace monitoring-prometheus + # 진짜 Discord webhook 노출 X — dummy URL. + # alertmanager 가 secret 마운트만 가능하면 검증 목표 (operator config 생성) 달성. + # reflector controller 는 CI 에 미설치 — 양쪽 namespace 에 직접 생성. + for ns in monitoring-grafana monitoring-prometheus; do + kubectl create secret generic discord-webhook \ + --namespace="$ns" \ + --from-literal=DISCORD_WEBHOOK_URL='https://discord.com/api/webhooks/0/dummy' + done + + - name: Install kube-prometheus-stack with prod values + if: steps.filter.outputs.monitoring == 'true' + run: | + set -uo pipefail + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + + # 환경별 호환 override: + # - grafana: 별도 chart 로 관리 (이 chart 의 grafana subchart 비활성) + # - ingress: cert-manager + 학교 DNS 의존 → CI 에서 비활성 + # - storageSpec: kind 의 local-path 와 충돌 방지 + helm install prom prometheus-community/kube-prometheus-stack \ + --version 79.5.0 \ + --namespace monitoring-prometheus \ + -f infra/k8s/monitoring/prometheus/values-production.yaml \ + --set grafana.enabled=false \ + --set prometheus.ingress.enabled=false \ + --set alertmanager.ingress.enabled=false \ + --set 'prometheus.prometheusSpec.storageSpec=null' \ + --wait --timeout 5m \ + || echo "::warning::helm install timed out — assertion 단계로 계속 진행 (silent fail 가능성)" + + - name: Wait for operator reconcile + if: steps.filter.outputs.monitoring == 'true' + run: sleep 30 + + - name: Assertion - prometheus-operator log에 silent reconcile fail 없음 + if: steps.filter.outputs.monitoring == 'true' + run: | + set -uo pipefail + OP_POD=$(kubectl get pod -n monitoring-prometheus \ + -l app=kube-prometheus-stack-operator -o name 2>/dev/null | head -1) + if [[ -z "$OP_POD" ]]; then + echo "::error::prometheus-operator pod not found" + exit 1 + fi + ERR_LOG=$(kubectl logs -n monitoring-prometheus "$OP_POD" --tail=300 2>&1 \ + | grep -E 'failed to (initialize|provision|sync)' || true) + if [[ -n "$ERR_LOG" ]]; then + echo "::error::prometheus-operator silent reconcile fail 감지" + echo "$ERR_LOG" + exit 1 + fi + echo "operator log clean" + + - name: Assertion - alertmanager-generated secret 에 우리 config 반영됨 + if: steps.filter.outputs.monitoring == 'true' + run: | + set -uo pipefail + GEN_SECRET="alertmanager-prom-kube-prometheus-stack-alertmanager-generated" + SECRET_CONTENT=$(kubectl get secret -n monitoring-prometheus "$GEN_SECRET" \ + -o jsonpath='{.data.alertmanager\.yaml\.gz}' 2>/dev/null \ + | base64 -d | gunzip 2>/dev/null || true) + if [[ -z "$SECRET_CONTENT" ]]; then + echo "::error::alertmanager generated secret 비어있음 — operator 가 secret 생성 못 함" + exit 1 + fi + if ! echo "$SECRET_CONTENT" | grep -q 'discord_configs'; then + echo "::error::discord_configs receiver 누락 — values.yaml 변경 미반영" + exit 1 + fi + if ! echo "$SECRET_CONTENT" | grep -q 'inhibit_rules'; then + echo "::error::inhibit_rules 누락 (alert storm 위험)" + exit 1 + fi + echo "alertmanager config 정상 반영" + + - name: Diagnostics on failure + if: failure() && steps.filter.outputs.monitoring == 'true' + run: | + echo "=== pods ===" + kubectl get pod -n monitoring-prometheus + echo + echo "=== alertmanager describe ===" + kubectl describe pod -n monitoring-prometheus -l app.kubernetes.io/name=alertmanager || true + echo + echo "=== operator logs (tail 200) ===" + OP_POD=$(kubectl get pod -n monitoring-prometheus \ + -l app=kube-prometheus-stack-operator -o name 2>/dev/null | head -1) + if [[ -n "$OP_POD" ]]; then + kubectl logs -n monitoring-prometheus "$OP_POD" --tail=200 || true + fi From a576b134532386b40387642e3bd8189d66282f93 Mon Sep 17 00:00:00 2001 From: JSON Date: Mon, 27 Apr 2026 17:13:47 +0900 Subject: [PATCH 2/2] feat(infra): add layered ci validation (L1-L5 + L7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a layered validation stack to ci-infra.yml that catches different classes of bugs at different cost tiers: L1 yaml-lint (blocking) — basic YAML syntax L2 kustomize-build (blocking) — overlay rendering, missing resources L3 helm-template (warn-only) — Go template + required values L4 kubeconform (warn-only) — CRD schema strict validation L5 component-dryrun (warn-only) — amtool/promtool/promtail per-tool L7 monitoring-e2e (blocking) — kind cluster + operator log assertion Hybrid blocking strategy: cheap layers (L1, L2) and the catch-all (L7) block merge from day 1; semi-noisy layers (L3-L5) start as warn-only to surface existing baseline violations without immediately blocking unrelated PRs. Promote to blocking after baseline cleanup PRs land. L5 amtool version is intentionally pinned to alertmanager 0.29.0 to match what prometheus-operator vendors. Mismatch causes false negatives of the exact silent-fail class this PR is meant to catch. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci-infra.yml | 253 ++++++++++++++++++++++++++++++++- 1 file changed, 252 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-infra.yml b/.github/workflows/ci-infra.yml index 8a38650e15..20858a7565 100644 --- a/.github/workflows/ci-infra.yml +++ b/.github/workflows/ci-infra.yml @@ -104,9 +104,260 @@ jobs: echo "All keys are valid" + yaml-lint: + name: L1 YAML lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Check if k8s manifests changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + k8s: + - 'infra/k8s/**' + - '.github/workflows/ci-infra.yml' + + - name: Install yamllint + if: steps.filter.outputs.k8s == 'true' + run: pip install --quiet yamllint + + - name: Lint + if: steps.filter.outputs.k8s == 'true' + run: | + yamllint -d "{extends: relaxed, rules: {line-length: disable, comments-indentation: disable}}" infra/k8s/ + + kustomize-build: + name: L2 Kustomize build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Check if k8s manifests changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + k8s: + - 'infra/k8s/**' + - '.github/workflows/ci-infra.yml' + + - name: Install kustomize + if: steps.filter.outputs.k8s == 'true' + run: | + curl -fsSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + sudo mv kustomize /usr/local/bin/ + + - name: Build all overlays + if: steps.filter.outputs.k8s == 'true' + shell: bash + run: | + set -uo pipefail + FAIL=0 + while IFS= read -r kfile; do + dir=$(dirname "$kfile") + echo "::group::$dir" + if ! kustomize build "$dir" > /dev/null 2>&1; then + echo "::error file=$kfile::kustomize build 실패" + kustomize build "$dir" 2>&1 | tail -10 + FAIL=1 + fi + echo "::endgroup::" + done < <(find infra/k8s -name kustomization.yaml -type f) + if [[ $FAIL -ne 0 ]]; then + echo "::error::kustomize build 실패한 overlay 있음" + exit 1 + fi + echo "모든 overlay 정상 빌드" + + helm-template: + name: L3 Helm template (warn-only) + runs-on: ubuntu-latest + continue-on-error: true # 베이스라인 정리 후 blocking 으로 승격 + steps: + - uses: actions/checkout@v6 + + - name: Check if helm values changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + helm: + - 'infra/k8s/monitoring/**' + - 'infra/k8s/argocd/**' + - '.github/workflows/ci-infra.yml' + + - name: Setup helm + if: steps.filter.outputs.helm == 'true' + uses: azure/setup-helm@v4 + with: + version: 'v3.19.2' + + - name: Render charts from ApplicationSet definitions + if: steps.filter.outputs.helm == 'true' + shell: bash + run: | + set -uo pipefail + # ApplicationSet에서 chart+version+valueFiles 추출 후 helm template + # (단순화: monitoring stack 의 핵심 chart만 우선. 다른 stack은 점진 추가) + declare -A CHARTS=( + ["prometheus-community/kube-prometheus-stack:79.5.0"]="infra/k8s/monitoring/prometheus/values-production.yaml" + ) + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts >/dev/null + helm repo update >/dev/null + FAIL=0 + for k in "${!CHARTS[@]}"; do + chart="${k%%:*}" + version="${k##*:}" + values="${CHARTS[$k]}" + echo "::group::$chart $version with $values" + if ! helm template test "$chart" --version "$version" -f "$values" > /dev/null 2>&1; then + echo "::error::helm template 실패: $chart $version" + helm template test "$chart" --version "$version" -f "$values" 2>&1 | tail -10 + FAIL=1 + fi + echo "::endgroup::" + done + [[ $FAIL -eq 0 ]] + + kubeconform: + name: L4 Kubeconform (warn-only) + runs-on: ubuntu-latest + continue-on-error: true # 베이스라인 정리 후 blocking 으로 승격 + needs: [kustomize-build] + steps: + - uses: actions/checkout@v6 + + - name: Check if k8s manifests changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + k8s: + - 'infra/k8s/**' + - '.github/workflows/ci-infra.yml' + + - name: Install kustomize + kubeconform + if: steps.filter.outputs.k8s == 'true' + run: | + curl -fsSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + sudo mv kustomize /usr/local/bin/ + curl -fsSL https://github.com/yannh/kubeconform/releases/latest/download/kubeconform-linux-amd64.tar.gz \ + | tar -xz -C /tmp kubeconform + sudo mv /tmp/kubeconform /usr/local/bin/ + + - name: Validate kustomize-built manifests against CRD schemas + if: steps.filter.outputs.k8s == 'true' + shell: bash + run: | + set -uo pipefail + FAIL=0 + while IFS= read -r kfile; do + dir=$(dirname "$kfile") + echo "::group::$dir" + if ! kustomize build "$dir" 2>/dev/null | kubeconform -strict \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + -summary 2>&1; then + echo "::warning file=$kfile::kubeconform 검증 실패 (warn-only)" + FAIL=1 + fi + echo "::endgroup::" + done < <(find infra/k8s -name kustomization.yaml -type f) + [[ $FAIL -eq 0 ]] + + component-dryrun: + name: L5 Component dry-run (warn-only) + runs-on: ubuntu-latest + continue-on-error: true # 베이스라인 정리 후 blocking 으로 승격 + steps: + - uses: actions/checkout@v6 + + - name: Check if monitoring source code has changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + monitoring: + - 'infra/k8s/monitoring/**' + - '.github/workflows/ci-infra.yml' + + - name: Setup helm + if: steps.filter.outputs.monitoring == 'true' + uses: azure/setup-helm@v4 + with: + version: 'v3.19.2' + + - name: amtool check-config (alertmanager 0.29.0 — operator vendored 매칭) + if: steps.filter.outputs.monitoring == 'true' + shell: bash + run: | + set -uo pipefail + # CAUTION: amtool 버전을 prometheus-operator vendored alertmanager 버전과 정확히 맞춰야 함 + # (mismatch 시 우리 PR #3543 같은 silent fail 못 잡음 — false negative) + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts >/dev/null + helm repo update >/dev/null + # alertmanager.config 만 추출 + helm template test prometheus-community/kube-prometheus-stack --version 79.5.0 \ + -f infra/k8s/monitoring/prometheus/values-production.yaml \ + --set grafana.enabled=false 2>/dev/null \ + | python3 -c " + import sys, yaml, base64, gzip + for doc in yaml.safe_load_all(sys.stdin): + if doc and doc.get('kind') == 'Alertmanager': + cfg = doc.get('spec', {}).get('config', '') + if cfg: + print(cfg) + sys.exit(0) + " > /tmp/am.yaml + if [[ ! -s /tmp/am.yaml ]]; then + echo "::warning::Alertmanager spec.config 추출 실패" + exit 0 + fi + docker run --rm -v /tmp:/cfg quay.io/prometheus/alertmanager:v0.29.0 \ + amtool check-config /cfg/am.yaml || { + echo "::warning::amtool check-config 실패 — Alertmanager config 문제 가능성" + exit 1 + } + + - name: promtool check rules (PrometheusRule) + if: steps.filter.outputs.monitoring == 'true' + shell: bash + run: | + set -uo pipefail + # 직접 정의한 PrometheusRule 파일 검증 (chart default 는 외부 검증 불필요) + FILES=$(find infra/k8s -name '*.yaml' -exec grep -l 'kind: PrometheusRule' {} \; 2>/dev/null || true) + if [[ -z "$FILES" ]]; then + echo "검증할 PrometheusRule 파일 없음" + exit 0 + fi + for f in $FILES; do + echo "::group::$f" + docker run --rm -v "$(pwd)/$f:/rules.yaml" prom/prometheus:v3.5.0 \ + promtool check rules /rules.yaml || true + echo "::endgroup::" + done + + - name: promtail -dry-run + if: steps.filter.outputs.monitoring == 'true' + shell: bash + run: | + set -uo pipefail + # promtail config 추출 후 dry-run + if [[ ! -f infra/k8s/monitoring/promtail/values.yaml ]]; then + echo "promtail values.yaml 없음 - skip" + exit 0 + fi + helm repo add grafana https://grafana.github.io/helm-charts >/dev/null + helm repo update >/dev/null + # promtail chart 자동 추출은 복잡 — 간단히 values.yaml 자체 검증만 + echo "::warning::promtail dry-run 자동화 미구현 (수동 검증 필요)" + monitoring-e2e: - name: Monitoring Stack E2E (kind) + name: L7 Monitoring Stack E2E (kind) runs-on: ubuntu-latest + needs: [yaml-lint, kustomize-build] steps: - uses: actions/checkout@v6