Merge main into dev/v0.33-windows

.azure-pipelines/perf.yml

@@ -0,0 +1,117 @@
+# retina Performance Test pipeline (Azure DevOps).
+#
+# Mirrors .github/workflows/perf-schedule.yaml + perf-manual.yaml + perf-template.yaml.
+# Scheduled: twice-daily at 00:17 and 12:17 UTC (both basic and advanced).
+# Manual: single mode chosen at queue time.
+#
+# Required ADO pipeline configuration (set in the ADO UI):
+#   Variables (plain):
+#     AZURE_LOCATION
+#     AZURE_AGENT_LINUX_SKU
+#     AZURE_AGENT_WINDOWS_SKU
+#     AZURE_AGENT_LINUX_ARM_SKU
+#   Variables (secret):
+#     AZURE_APP_INSIGHTS_KEY
+#   Service connection (Azure RM, WIF): r2d
+#   Settings: "Limit building pull requests from forked GitHub repositories" = Disable
+
+trigger: none
+pr: none
+
+schedules:
+  - cron: '17 0,12 * * *'
+    displayName: Twice-daily perf test (00:17 and 12:17 UTC)
+    branches:
+      include:
+        - main
+    always: true
+
+parameters:
+  - name: image_registry
+    displayName: Image Registry
+    type: string
+    default: ghcr.io
+  - name: image_namespace
+    displayName: Image Namespace
+    type: string
+    default: microsoft/retina
+  - name: tag
+    displayName: Image Tag (blank = latest GitHub release tag)
+    type: string
+    default: ' '
+  - name: modes
+    displayName: Retina modes to run
+    type: object
+    default:
+      - basic
+      - advanced
+
+variables:
+  - name: azureServiceConnection
+    value: r2d
+
+jobs:
+  - ${{ each mode in parameters.modes }}:
+      - job: perf_${{ mode }}
+        displayName: Retina ${{ mode }} Performance Test
+        timeoutInMinutes: 150
+        pool:
+          vmImage: ubuntu-latest
+        variables:
+          CLUSTER_NAME: retina-perf-${{ mode }}-$(Build.BuildId)-$(System.JobAttempt)
+          MODE: ${{ mode }}
+        steps:
+          - checkout: self
+          - template: steps/setup-go.yml
+          - task: AzureCLI@2
+            displayName: Resolve image tag
+            name: resolve_tag
+            inputs:
+              azureSubscription: $(azureServiceConnection)
+              scriptType: bash
+              scriptLocation: inlineScript
+              inlineScript: |
+                set -euo pipefail
+                TAG="${{ parameters.tag }}"
+                TAG="${TAG// /}"
+                if [[ -z "$TAG" ]]; then
+                  TAG=$(curl -s https://api.github.com/repos/microsoft/retina/releases/latest | jq -r .tag_name)
+                fi
+                echo "##vso[task.setvariable variable=RESOLVED_TAG]$TAG"
+                echo "Resolved tag: $TAG"
+          - task: AzureCLI@2
+            displayName: Run performance measurement for Retina
+            env:
+              AZURE_APP_INSIGHTS_KEY: $(AZURE_APP_INSIGHTS_KEY)
+              AZURE_LOCATION: $(AZURE_LOCATION)
+              AZURE_AGENT_LINUX_SKU: $(AZURE_AGENT_LINUX_SKU)
+              AZURE_AGENT_WINDOWS_SKU: $(AZURE_AGENT_WINDOWS_SKU)
+              AZURE_AGENT_LINUX_ARM_SKU: $(AZURE_AGENT_LINUX_ARM_SKU)
+              TAG: $(RESOLVED_TAG)
+              REGISTRY: ${{ parameters.image_registry }}
+              NAMESPACE: ${{ parameters.image_namespace }}
+            inputs:
+              azureSubscription: $(azureServiceConnection)
+              scriptType: bash
+              scriptLocation: inlineScript
+              inlineScript: |
+                set -euo pipefail
+                export AZURE_SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+                echo "Running in $MODE mode with tag $TAG..."
+                go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args \
+                  -image-tag="$TAG" \
+                  -image-registry="$REGISTRY" \
+                  -image-namespace="$NAMESPACE" \
+                  -retina-mode="$MODE"
+          - task: AzureCLI@2
+            displayName: Cleanup resource group
+            condition: always()
+            inputs:
+              azureSubscription: $(azureServiceConnection)
+              scriptType: bash
+              scriptLocation: inlineScript
+              inlineScript: |
+                if az group exists --name "$(CLUSTER_NAME)" 2>/dev/null | grep -q true; then
+                  echo "Deleting resource group $(CLUSTER_NAME)..."
+                  az group delete --name "$(CLUSTER_NAME)" --yes --no-wait || true
+                fi

.azure-pipelines/scale.yml

@@ -0,0 +1,114 @@
+# retina Scale Test pipeline (Azure DevOps).
+#
+# Mirrors .github/workflows/scale-test.yaml + daily-scale-test.yaml.
+# Daily scheduled run (00:00 UTC) + manual runs from ADO UI.
+#
+# Required ADO pipeline configuration (set in the ADO UI):
+#   Variables (plain):
+#     ACR_NAME
+#   Variables (secret):
+#     AZURE_APP_INSIGHTS_KEY
+#   Service connection (Azure RM, WIF): r2d
+#   Settings: "Limit building pull requests from forked GitHub repositories" = Disable
+
+trigger: none
+pr: none
+
+schedules:
+  - cron: '0 0 * * *'
+    displayName: Daily scale test (00:00 UTC)
+    branches:
+      include:
+        - main
+    always: true
+
+parameters:
+  # For manual runs from the ADO UI. Defaults match .github/workflows/daily-scale-test.yaml.
+  - name: resource_group
+    displayName: Azure Resource Group (leave blank to auto-create)
+    type: string
+    default: ' '
+  - name: cluster_name
+    displayName: AKS Cluster Name (leave blank to auto-create)
+    type: string
+    default: ' '
+  - name: image_namespace
+    displayName: Image Namespace
+    type: string
+    default: microsoft/retina
+  - name: image_tag
+    displayName: Image Tag (blank = latest commit from main)
+    type: string
+    default: ' '
+  - name: num_deployments
+    displayName: Number of Traffic Deployments
+    type: number
+    default: 1000
+  - name: num_replicas
+    displayName: Number of Traffic Replicas per Deployment
+    type: number
+    default: 20
+  - name: num_netpol
+    displayName: Number of Network Policies
+    type: number
+    default: 0
+  - name: num_nodes
+    displayName: Number of nodes
+    type: number
+    default: 1000
+  - name: cleanup
+    displayName: Clean up environment after test
+    type: boolean
+    default: true
+
+variables:
+  - name: azureServiceConnection
+    value: r2d
+
+jobs:
+  - job: scale_test
+    displayName: Scale Test
+    timeoutInMinutes: 360
+    pool:
+      vmImage: ubuntu-latest
+    steps:
+      - checkout: self
+      - template: steps/setup-go.yml
+      - task: AzureCLI@2
+        displayName: Run Scale Test
+        env:
+          AZURE_RESOURCE_GROUP: ${{ parameters.resource_group }}
+          CLUSTER_NAME: ${{ parameters.cluster_name }}
+          NUM_DEPLOYMENTS: ${{ parameters.num_deployments }}
+          NUM_REPLICAS: ${{ parameters.num_replicas }}
+          NUM_NETPOLS: ${{ parameters.num_netpol }}
+          CLEANUP: ${{ parameters.cleanup }}
+          IMAGE_NAMESPACE: ${{ parameters.image_namespace }}
+          TAG: ${{ parameters.image_tag }}
+          NODES: ${{ parameters.num_nodes }}
+          AZURE_APP_INSIGHTS_KEY: $(AZURE_APP_INSIGHTS_KEY)
+        inputs:
+          azureSubscription: $(azureServiceConnection)
+          scriptType: bash
+          scriptLocation: inlineScript
+          inlineScript: |
+            set -euo pipefail
+            export AZURE_SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+            export IMAGE_REGISTRY=$(ACR_NAME)
+            # Trim placeholder whitespace defaults (ADO string params cannot be truly empty).
+            TAG="${TAG// /}"
+            AZURE_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP// /}"
+            CLUSTER_NAME="${CLUSTER_NAME// /}"
+            # Default TAG to the value make version would produce for this commit,
+            # keeping it consistent with the tag used by all image-build targets.
+            [[ -z "$TAG" ]] && TAG="$(make version)"
+            # Scheduled runs create infra; manual runs with a provided cluster reuse it.
+            if [[ "$(Build.Reason)" == "Manual" && -n "$CLUSTER_NAME" ]]; then
+              CREATE_INFRA=false
+            else
+              CREATE_INFRA=true
+            fi
+            export TAG AZURE_RESOURCE_GROUP CLUSTER_NAME
+            exec go test -v ./test/e2e/. -timeout 300m -tags=scale -count=1 -args \
+              -create-infra=$CREATE_INFRA \
+              -delete-infra=$CLEANUP

.azure-pipelines/steps/free-disk-space.yml

@@ -0,0 +1,31 @@
+# Free ~30-35 GB on Microsoft-hosted Ubuntu agents by removing pre-installed
+# toolchains. Needed for large image builds (retina-agent especially) where
+# the go build cache in /tmp overflows the root partition's ~14 GB free space.
+#
+# Notes on what is / isn't removed:
+#   Removed:      .NET SDK, Android SDK, Haskell (GHC), Boost, PowerShell,
+#                 Swift, node_modules — none of which retina builds use.
+#   Kept:         CodeQL toolcache (used by other pipelines in the repo),
+#                 hostedtoolcache/go (used by GoTool@0 to install Go).
+#   Docker prune: clears any preloaded images / build cache from the agent.
+#
+# Runs on every architecture — even native amd64 image builds need the space
+# once the go build cache grows to include every retina + plugin dependency.
+steps:
+  - bash: |
+      set -e
+      echo "=== Before cleanup ==="
+      df -h /
+      sudo rm -rf \
+        /usr/share/dotnet \
+        /usr/local/lib/android \
+        /opt/ghc \
+        /usr/local/share/boost \
+        /usr/local/share/powershell \
+        /opt/microsoft/powershell \
+        /usr/share/swift \
+        /usr/local/lib/node_modules
+      sudo docker system prune -af --volumes || true
+      echo "=== After cleanup ==="
+      df -h /
+    displayName: Free disk space

.azure-pipelines/steps/setup-go.yml

@@ -0,0 +1,15 @@
+# Install the Go version declared in the repo's go.mod.
+# ADO's GoTool@0 does not support "from go.mod" (unlike GitHub Actions'
+# setup-go@v6), so we parse it explicitly and pass the value.
+#
+# Assumes `checkout: self` has already run in the calling job.
+steps:
+  - bash: |
+      set -euo pipefail
+      GO_VER=$(awk '/^go [0-9]/ {print $2; exit}' go.mod)
+      echo "Detected Go version from go.mod: $GO_VER"
+      echo "##vso[task.setvariable variable=GO_VERSION_FROM_GO_MOD]$GO_VER"
+    displayName: Read Go version from go.mod
+  - task: GoTool@0
+    inputs:
+      version: $(GO_VERSION_FROM_GO_MOD)

.devcontainer/devcontainer.json

@@ -1,25 +1,102 @@
 {
   "name": "retina",
-  "image": "mcr.microsoft.com/devcontainers/base:jammy",
+  "image": "mcr.microsoft.com/devcontainers/base:noble",
   "features": {
-    "ghcr.io/devcontainers/features/common-utils:2": {},
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {},
-    "ghcr.io/devcontainers/features/github-cli:1": {},
-    "ghcr.io/devcontainers/features/go:1": {},
-    "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {},
-    "ghcr.io/devcontainers-contrib/features/kind:1": {},
-    "ghcr.io/devcontainers/features/azure-cli:1": {}
+    "ghcr.io/devcontainers/features/docker-in-docker:2.16.1": {},
+    "ghcr.io/devcontainers/features/github-cli:1.1.0": {},
+    "ghcr.io/devcontainers/features/go:1.3.4": {
+      "version": "1.24.11"
+    },
+    "ghcr.io/devcontainers/features/kubectl-helm-minikube:1.3.1": {},
+    "ghcr.io/devcontainers-extra/features/kind:1.0.15": {},
+    "ghcr.io/devcontainers/features/azure-cli:1.2.9": {},
+    // LLVM 17 is the minimum version available for Ubuntu Noble on apt.llvm.org.
+    // Provides clang and llvm-strip needed for eBPF compilation.
+    "ghcr.io/devcontainers-community/features/llvm:3.2.0": {
+      "version": "17"
+    }
+  },
+  "hostRequirements": {
+    "cpus": 4,
+    "memory": "8gb",
+    "storage": "32gb"
+  },
+  // Persist Go module and build caches across container rebuilds.
+  "mounts": [
+    {
+      "type": "volume",
+      "source": "retina-gomodcache",
+      "target": "/go/pkg/mod"
+    },
+    {
+      "type": "volume",
+      "source": "retina-gobuildcache",
+      "target": "/home/vscode/.cache/go-build"
+    }
+  ],
+  // These commands run in parallel during container creation.
+  "onCreateCommand": {
+    // The LLVM feature installs versioned binaries (clang-17, llvm-strip-17).
+    // Create unversioned symlinks so the build system can find them.
+    "symlinks": "sudo ln -sf /usr/bin/clang-17 /usr/bin/clang && sudo ln -sf /usr/bin/llvm-strip-17 /usr/bin/llvm-strip",
+    // Fix ownership of volume mounts (created as root) and cache dirs,
+    // then download Go modules.
+    "go-setup": "sudo chown -R vscode:vscode /go /home/vscode/.cache && go mod download",
+    // Install jq (needed by some Makefile targets and scripts).
+    "apt-deps": "sudo apt-get update && sudo apt-get install -y --no-install-recommends jq && sudo rm -rf /var/lib/apt/lists/*"
+  },
+  // Wait for Docker-in-Docker to be ready, then create a Kind cluster for local testing.
+  "postStartCommand": {
+    "kind": "while ! docker info >/dev/null 2>&1; do sleep 1; done && kind create cluster 2>/dev/null || true"
+  },
+  "waitFor": "onCreateCommand",
+  "forwardPorts": [
+    9965,
+    4244,
+    10093
+  ],
+  "portsAttributes": {
+    "9965": {
+      "label": "Hubble Metrics",
+      "onAutoForward": "silent"
+    },
+    "4244": {
+      "label": "Hubble Relay",
+      "onAutoForward": "silent"
+    },
+    "10093": {
+      "label": "Retina Metrics",
+      "onAutoForward": "silent"
+    }
   },
-  "postCreateCommand": "bash .devcontainer/installMoreTools.sh && kind create cluster",
   "customizations": {
     "vscode": {
       "extensions": [
-        "esbenp.prettier-vscode",
         "golang.go",
-        "mutantdino.resourcemonitor",
         "ms-vscode.makefile-tools",
-        "ms-kubernetes-tools.vscode-kubernetes-tools"
-      ]
+        "ms-kubernetes-tools.vscode-kubernetes-tools",
+        "ms-azuretools.vscode-docker",
+        "redhat.vscode-yaml",
+        "mutantdino.resourcemonitor",
+        "DavidAnson.vscode-markdownlint"
+      ],
+      "settings": {
+        "go.lintTool": "golangci-lint",
+        "go.lintFlags": [
+          "--config=.golangci.yaml",
+          "--timeout=10m"
+        ],
+        "go.formatTool": "gofumpt",
+        "files.insertFinalNewline": true,
+        "markdownlint.config": {
+          "extends": ".github/.markdownlint.json"
+        },
+        "[markdown]": {
+          "editor.codeActionsOnSave": {
+            "source.fixAll.markdownlint": "explicit"
+          }
+        }
+      }
     }
   }
 }