OPNET-803: installer: add BGP VIP coexistence lanes (OVN-K route advertisements, day-2 MetalLB)

ci-operator/config/openshift/installer/openshift-installer-main.yaml

@@ -857,6 +857,14 @@ tests:
   steps:
     cluster_profile: equinix-ocp-metal
     workflow: baremetalds-e2e-bgp-vip
+- always_run: false
+  as: e2e-metal-ipi-bgp-vip-ovn-bgp
+  capabilities:
+  - intranet
+  optional: true
+  steps:
+    cluster_profile: equinix-ocp-metal
+    workflow: baremetalds-e2e-bgp-vip-ovn-bgp
 - always_run: false
   as: e2e-metal-ipi-ovn-swapped-hosts
   capabilities:

ci-operator/jobs/openshift/installer/openshift-installer-main-presubmits.yaml

@@ -7454,6 +7454,96 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-metal-ipi-bgp-vip,?($|\s.*)
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build03
+    context: ci/prow/e2e-metal-ipi-bgp-vip-ovn-bgp
+    decorate: true
+    decoration_config:
+      sparse_checkout_files:
+      - .ci-operator.yaml
+      - images/baremetal/Dockerfile.ci
+      - images/installer/Dockerfile.ci
+      - images/installer/Dockerfile.upi.ci
+      - images/libvirt/Dockerfile.ci
+      - images/openstack/Dockerfile.ci
+    labels:
+      capability/intranet: intranet
+      ci-operator.openshift.io/cloud: equinix-ocp-metal
+      ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-installer-main-e2e-metal-ipi-bgp-vip-ovn-bgp
+    optional: true
+    rerun_command: /test e2e-metal-ipi-bgp-vip-ovn-bgp
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=e2e-metal-ipi-bgp-vip-ovn-bgp
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-metal-ipi-bgp-vip-ovn-bgp,?($|\s.*)
   - agent: kubernetes
     always_run: false
     annotations:

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+- mkowalski
+- cybertron
+reviewers:
+- mkowalski
+- cybertron

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/baremetalds-e2e-bgp-vip-ovn-bgp-workflow.metadata.json

@@ -0,0 +1,13 @@
+{
+	"path": "baremetalds/e2e/bgp-vip/ovn-bgp/baremetalds-e2e-bgp-vip-ovn-bgp-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"mkowalski",
+			"cybertron"
+		],
+		"reviewers": [
+			"mkowalski",
+			"cybertron"
+		]
+	}
+}

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/baremetalds-e2e-bgp-vip-ovn-bgp-workflow.yaml

@@ -0,0 +1,40 @@
+workflow:
+  as: baremetalds-e2e-bgp-vip-ovn-bgp
+  steps:
+    cluster_profile: equinix-ocp-metal
+    env:
+      DEVSCRIPTS_CONFIG: |
+        IP_STACK=v4
+        NUM_WORKERS=2
+        ENABLE_BGP_TOR=true
+        BGP_VIP_MANAGEMENT=true
+      FEATURE_SET: DevPreviewNoUpgrade
+      EXTRA_MG_ARGS: --host-network
+      TEST_SKIPS: Services should fallback to local terminating endpoints when there
+        are no ready endpoints with externalTrafficPolicy=Local\| CPU Partitioning
+        cluster platform workloads should be annotated correctly for Deployments
+    pre:
+      - ref: baremetalds-devscripts-conf-featureset
+      - chain: baremetalds-ofcir-pre
+      - ref: baremetalds-e2e-ovn-bgp-pre
+    test:
+      - ref: baremetalds-e2e-bgp-vip-verify
+      - ref: baremetalds-e2e-bgp-vip-ovn-bgp-verify
+      - chain: baremetalds-ipi-test
+    post:
+      - chain: baremetalds-ofcir-post
+  documentation: |-
+    Coexistence lane for BGP-based VIP management (enhancement 1982) and
+    OVN-Kubernetes BGP route advertisements. Installs a baremetal IPI cluster
+    via dev-scripts with BGP_VIP_MANAGEMENT=true (API/ingress VIPs advertised
+    to a top-of-rack FRR speaker by kube-vip + frr-k8s static pods instead of
+    keepalived), then runs the OVN BGP pre step: an external FRR route
+    reflector on the baremetal bridge, routeAdvertisements enabled on the
+    cluster network operator, and FRRConfiguration/RouteAdvertisements CRs for
+    the default pod network. The verification steps assert that both BGP
+    consumers work simultaneously: the VIP acceptance criteria still hold, the
+    RouteAdvertisements-generated FRRConfiguration is merged by the frr-k8s
+    static pods on the control plane (where the frr-k8s DaemonSet does not
+    run), every node establishes a session to the route reflector, and pod
+    network prefixes arrive at the reflector; the conformance suite then runs
+    against the combined configuration.

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/verify/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+- mkowalski
+- cybertron
+reviewers:
+- mkowalski
+- cybertron

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/verify/baremetalds-e2e-bgp-vip-ovn-bgp-verify-commands.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+set -o nounset
+set -o errexit
+set -o pipefail
+
+echo "************ baremetalds bgp-vip ovn-bgp coexistence verify command ************"
+
+# Fetch packet basic configuration
+# shellcheck source=/dev/null
+source "${SHARED_DIR}/packet-conf.sh"
+
+ssh "${SSHOPTS[@]}" "root@${IP}" bash -x - << 'EOF'
+#!/bin/bash
+set -o nounset
+set -o errexit
+set -o pipefail
+set -x
+
+export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig
+
+CLI="podman"
+if ! command -v podman &>/dev/null; then
+    CLI="docker"
+fi
+
+FAILURES=0
+fail() {
+    echo "FAIL: $*"
+    FAILURES=$((FAILURES + 1))
+}
+
+# poll <deadline-seconds> <function> — re-evaluate until success or timeout,
+# allowing normal CR/BGP reconciliation to converge before asserting state
+poll() {
+    local deadline=$((SECONDS + $1)); shift
+    until "$@"; do
+        if (( SECONDS >= deadline )); then
+            return 1
+        fi
+        sleep 10
+    done
+}
+
+nodes="$(oc get nodes -o name | wc -l)"
+
+echo "[1/4] both BGP consumers own FRRConfiguration CRs in openshift-frr-k8s"
+check_crs() {
+    oc get frrconfiguration -n openshift-frr-k8s bgp-vip receive-filtered &>/dev/null
+}
+if ! poll 120 check_crs; then
+    oc get frrconfiguration -n openshift-frr-k8s || true
+    fail "FRRConfigurations 'bgp-vip' and 'receive-filtered' not both present in openshift-frr-k8s"
+fi
+
+echo "[2/4] every node has an Established BGP session to the route reflector"
+# The route reflector (external 'frr' container, 192.168.111.3) peers with
+# every node. On the control plane the RouteAdvertisements-generated
+# FRRConfiguration must be merged by the frr-k8s *static pods* (the frr-k8s
+# DaemonSet does not run there), so established master sessions prove the
+# static-pod CR merge works alongside the VIP configuration.
+check_sessions() {
+    local established
+    established="$(${CLI} exec frr vtysh -c 'show bgp ipv4 unicast summary json' \
+        | jq '[.peers[] | select(.state=="Established")] | length')"
+    [[ "${established:-0}" -eq "${nodes}" ]]
+}
+if ! poll 300 check_sessions; then
+    ${CLI} exec frr vtysh -c 'show bgp ipv4 unicast summary' || true
+    fail "route reflector does not have ${nodes} Established sessions (one per node)"
+fi
+
+echo "[3/4] every node's pod subnet is advertised to the route reflector"
+# assert the exact per-node OVN subnets, not a route count: the reflector's
+# table also carries unrelated prefixes (e.g. the agnhost network)
+check_pod_subnets() {
+    local rr_routes subnet missing=0
+    rr_routes="$(${CLI} exec frr vtysh -c 'show bgp ipv4 unicast json' | jq -r '.routes | keys[]')"
+    for subnet in $(oc get nodes -o jsonpath='{.items[*].metadata.annotations.k8s\.ovn\.org/node-subnets}' \
+        | jq -r -s '.[].default[]' | grep -F . ); do
+        if ! grep -qx "${subnet}" <<< "${rr_routes}"; then
+            echo "pod subnet ${subnet} not (yet) at the route reflector"
+            missing=1
+        fi
+    done
+    [[ "${missing}" -eq 0 ]]
+}
+if ! poll 300 check_pod_subnets; then
+    ${CLI} exec frr vtysh -c 'show bgp ipv4 unicast' || true
+    fail "not every node's pod subnet is advertised to the route reflector"
+fi
+
+echo "[4/4] pod-network datapath over BGP: pod reaches the external agnhost"
+# 172.20.0.100 lives behind the route reflector (agnhost macvlan network);
+# the cluster imports it via the receive-filtered FRRConfiguration. A pod
+# reaching it proves the RA datapath works on a BGP-VIP-managed cluster.
+oc delete pod bgp-ra-datapath-check --ignore-not-found
+if oc run bgp-ra-datapath-check --restart=Never --attach --rm --pod-running-timeout=5m \
+    --image=registry.k8s.io/e2e-test-images/agnhost:2.53 --command -- \
+    curl --max-time 20 -s --fail --show-error http://172.20.0.100:8000/hostname; then
+    echo "agnhost reachable from pod network"
+else
+    fail "pod could not reach agnhost 172.20.0.100:8000 over the BGP-imported route"
+fi
+
+if [[ "${FAILURES}" -ne 0 ]]; then
+    echo "BGP VIP + OVN-K route advertisements coexistence verification failed with ${FAILURES} error(s)"
+    exit 1
+fi
+echo "BGP VIP + OVN-K route advertisements coexistence verification passed"
+EOF

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/verify/baremetalds-e2e-bgp-vip-ovn-bgp-verify-ref.metadata.json

@@ -0,0 +1,13 @@
+{
+	"path": "baremetalds/e2e/bgp-vip/ovn-bgp/verify/baremetalds-e2e-bgp-vip-ovn-bgp-verify-ref.yaml",
+	"owners": {
+		"approvers": [
+			"mkowalski",
+			"cybertron"
+		],
+		"reviewers": [
+			"mkowalski",
+			"cybertron"
+		]
+	}
+}

ci-operator/step-registry/baremetalds/e2e/bgp-vip/ovn-bgp/verify/baremetalds-e2e-bgp-vip-ovn-bgp-verify-ref.yaml

@@ -0,0 +1,17 @@
+ref:
+  as: baremetalds-e2e-bgp-vip-ovn-bgp-verify
+  from: dev-scripts
+  commands: baremetalds-e2e-bgp-vip-ovn-bgp-verify-commands.sh
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  grace_period: 10m
+  documentation: |-
+    Verifies coexistence of BGP-based VIP management (enhancement 1982) and
+    OVN-Kubernetes BGP route advertisements on the same cluster: both
+    consumers' FRRConfiguration CRs exist, every node (including control
+    plane nodes served by frr-k8s static pods rather than the DaemonSet)
+    establishes a BGP session to the external route reflector, per-node pod
+    subnets are advertised to the reflector, and a pod reaches an external
+    endpoint over a BGP-imported route.

ci-operator/step-registry/baremetalds/e2e/ovn/bgp/pre/baremetalds-e2e-ovn-bgp-pre-commands.sh

@@ -261,7 +261,9 @@ fi
 deploy_frr_external_container vrf_neighbors
 
 # apply FRR-K8s overrides to enable debug logging
-oc create namespace openshift-frr-k8s
+# (idempotent: the namespace may already exist when frr-k8s is pre-deployed,
+# e.g. by BGP-based VIP management static pods)
+oc create namespace openshift-frr-k8s --dry-run=client -o yaml | oc apply -f -
 oc apply -f - <<EOF
 kind: ConfigMap
 apiVersion: v1
@@ -275,6 +277,22 @@ EOF
 # enable route advertisement with FRR
 oc patch Network.operator.openshift.io cluster --type=merge -p='{"spec":{"additionalRoutingCapabilities": {"providers": ["FRR"]}, "defaultNetwork":{"ovnKubernetesConfig":{"routeAdvertisements":"Enabled"}}}}'
 
+# The RouteAdvertisements CRD is created by CNO while reconciling the
+# routeAdvertisements enablement. On clusters where frr-k8s is already
+# deployed (e.g. BGP-based VIP management) the daemonset rollout waits below
+# return immediately, so wait for the CRD explicitly (and before a possible
+# FRR_IMAGE managementState=Unmanaged transition stops CNO reconciliation).
+echo "Waiting for the RouteAdvertisements CRD..."
+crd_deadline=$((SECONDS + 600))
+until oc wait --for condition=Established crd/routeadvertisements.k8s.ovn.org --timeout 10s &> /dev/null; do
+  if (( SECONDS >= crd_deadline )); then
+    oc get crd/routeadvertisements.k8s.ovn.org -o yaml || true
+    echo "Timed out waiting for routeadvertisements.k8s.ovn.org" >&2
+    exit 1
+  fi
+  sleep 5
+done
+
 echo "Waiting for daemonset 'frr-k8s' to be created..."
 until oc rollout status daemonset -n openshift-frr-k8s frr-k8s --timeout 2m &> /dev/null; do
   sleep 5