feat: add foundational control plane support for Spock 6 clusters

.circleci/config.yml

@@ -137,6 +137,31 @@ jobs:
           when: always
       - store_test_results:
           path: .
+  test_e2e_spock6:
+    executor: common
+    steps:
+      - common_setup
+      - run:
+          name: Run Spock 6 add-node e2e test against the latest dev image
+          command: |
+            make ci-compose-detached
+            make test-e2e E2E_DEBUG=1 E2E_FIXTURE=ci E2E_RUN='^TestSpock6AddNode$$' TEST_RERUN_FAILS=2
+      - run:
+          name: Archive debug output
+          command: |
+            if [[ -d ./e2e/debug ]]; then
+              sudo journalctl -u docker.service > ./e2e/debug/docker-service.log
+              tar -czf e2e-debug.tar.gz -C e2e debug
+            fi
+          when: on_fail
+      - store_artifacts:
+          path: e2e-debug.tar.gz
+      - run:
+          name: Ensure Docker Compose is stopped
+          command: make ci-compose-down
+          when: always
+      - store_test_results:
+          path: .
   release:
     executor: common
     steps:
@@ -313,3 +338,22 @@ workflows:
     jobs:
       - build_image:
           context: control-plane-release
+
+  weekly_spock6:
+    # Runs independent of any commit activity, so drift introduced by an
+    # upstream Spock 6 build gets caught even if nobody touches this repo
+    # that week. Weekly rather than nightly: the upstream dev image only
+    # gets a new build every few weeks in practice, so nightly would just
+    # be ~20+ no-op runs for every one that actually catches something.
+    # The e2e test itself points at the floating spock6DevImage tag (see
+    # e2e/spock6_add_node_test.go), so this job needs no extra plumbing
+    # to track "latest" - only the schedule.
+    triggers:
+      - schedule:
+          cron: "0 6 * * 1"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - test_e2e_spock6

e2e/custom_db_create_test.go

@@ -5,8 +5,8 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"log"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -98,7 +98,7 @@ func TestCreateDbWithVersions(t *testing.T) {
 					Password: password,
 				}
 
-				verifyPgVersion(ctx, db, primaryOpts, version.PostgresVersion, t)
+				verifyPgVersion(ctx, db, primaryOpts, version.PostgresVersion, version.SpockVersion, t)
 				verifyPrimaryNodes(ctx, db, primaryOpts, t)
 			}
 
@@ -110,7 +110,7 @@ func TestCreateDbWithVersions(t *testing.T) {
 					Username: username,
 					Password: password,
 				}
-				verifyPgVersion(ctx, db, connOpts, version.PostgresVersion, t)
+				verifyPgVersion(ctx, db, connOpts, version.PostgresVersion, version.SpockVersion, t)
 				verifyReplicasNodes(ctx, db, connOpts, t)
 			}
 
@@ -206,19 +206,30 @@ func verifyReplicasNodes(ctx context.Context, db *DatabaseFixture,
 	})
 }
 
-// Validate postgresql version
+// Validate postgresql version. Spock 6 manifest entries point at a
+// floating/mutable dev image tag (see version-manifest.json), so their
+// resolved Postgres minor can drift past the declared version at any
+// time - only the major version is checked for those. Pinned versions
+// (Spock <= 5) are still checked for an exact match.
 func verifyPgVersion(ctx context.Context, db *DatabaseFixture,
-	primaryOpts ConnectionOptions, expectedVersion string, t testing.TB) {
+	primaryOpts ConnectionOptions, expectedVersion string, spockVersion string, t testing.TB) {
 	db.WithConnection(ctx, primaryOpts, t, func(conn *pgx.Conn) {
 		var versionStr string
 		err := conn.QueryRow(ctx, "SELECT version()").Scan(&versionStr)
 		if err != nil {
-			log.Fatalf("Failed to fetch PostgreSQL version: %v", err)
+			t.Fatalf("Failed to fetch PostgreSQL version: %v", err)
 		}
-		if !strings.Contains(versionStr, expectedVersion) {
-			log.Fatalf("Expected PostgreSQL version %s, but got: %s", expectedVersion, versionStr)
+
+		versionToMatch := expectedVersion
+		spockMajorStr, _, _ := strings.Cut(spockVersion, ".")
+		if spockMajor, err := strconv.Atoi(spockMajorStr); err == nil && spockMajor >= 6 {
+			versionToMatch, _, _ = strings.Cut(expectedVersion, ".")
+		}
+
+		if !strings.Contains(versionStr, versionToMatch) {
+			t.Fatalf("Expected PostgreSQL version %s, but got: %s", versionToMatch, versionStr)
 		}
-		tLogf(t, "PostgreSQL version validation passed (found %s)\n", expectedVersion)
+		tLogf(t, "PostgreSQL version validation passed (found %s)\n", versionStr)
 	})
 }
 

e2e/spock6_add_node_test.go

@@ -0,0 +1,100 @@
+//go:build e2e_test
+
+package e2e
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	controlplane "github.com/pgEdge/control-plane/api/apiv1/gen/control_plane"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// spock6DevImage is a floating/mutable tag tracking the latest Spock 6
+// development build. Not pinned to a specific build number: the scheduled
+// CI job re-running this test picks up whatever the tag currently resolves
+// to, with no extra plumbing needed to point CI at "latest."
+const spock6DevImage = "ghcr.io/pgedge/pgedge-postgres:18-spock6-standard"
+
+// TestSpock6AddNode validates the add-node workflow end-to-end against a
+// real Spock 6 cluster: creates a 2-node database pinned to a Spock 6 dev
+// image via orchestrator_opts.swarm.image (bypassing manifest version
+// constraints, since spock6 manifest entries are deliberately "dev"
+// stability and never auto-selected), adds a 3rd node, and confirms the
+// full mesh reaches "replicating" — exercising the Spock-major-gated
+// spock.progress query (PeerCatchupResource).
+func TestSpock6AddNode(t *testing.T) {
+	t.Parallel()
+
+	const (
+		username = "admin"
+		password = "password"
+		dbName   = "spock6_add_node_db"
+	)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
+	defer cancel()
+
+	hostIDs := fixture.HostIDs()
+
+	nodeSpec := func(name, hostID string) *controlplane.DatabaseNodeSpec {
+		return &controlplane.DatabaseNodeSpec{
+			Name:    name,
+			HostIds: []controlplane.Identifier{controlplane.Identifier(hostID)},
+			OrchestratorOpts: &controlplane.OrchestratorOpts{
+				Swarm: &controlplane.SwarmOpts{Image: pointerTo(spock6DevImage)},
+			},
+		}
+	}
+
+	t.Log("Step 1: Creating 2-node Spock 6 database fixture")
+	db := fixture.NewDatabaseFixture(ctx, t, &controlplane.CreateDatabaseRequest{
+		Spec: &controlplane.DatabaseSpec{
+			DatabaseName:    dbName,
+			PostgresVersion: pointerTo("18.6"),
+			SpockVersion:    pointerTo("6"),
+			Port:            pointerTo(0),
+			PatroniPort:     pointerTo(0),
+			DatabaseUsers: []*controlplane.DatabaseUserSpec{{
+				Username:   username,
+				Password:   pointerTo(password),
+				DbOwner:    pointerTo(true),
+				Attributes: []string{"LOGIN", "SUPERUSER"},
+			}},
+			Nodes: []*controlplane.DatabaseNodeSpec{
+				nodeSpec("n1", hostIDs[0]),
+				nodeSpec("n2", hostIDs[1]),
+			},
+		},
+	})
+	t.Logf("Database created: %s", db.ID)
+
+	t.Log("Step 2: Adding n3 node with n1 as source")
+	db.Spec.Nodes = append(db.Spec.Nodes, func() *controlplane.DatabaseNodeSpec {
+		n := nodeSpec("n3", hostIDs[2])
+		n.SourceNode = pointerTo("n1")
+		return n
+	}())
+	require.NoError(t, db.Update(ctx, UpdateOptions{Spec: db.Spec}))
+	t.Log("Add-node completed successfully against Spock 6")
+
+	t.Log("Step 3: Waiting for full mesh replication")
+	db.WaitForReplication(ctx, t, username, password)
+	t.Log("Replication complete")
+
+	t.Log("Step 4: Verifying spock.spock_version() reports major 6 on the new node")
+	n3Opts := ConnectionOptions{
+		Matcher:  And(WithNode("n3"), WithRole("primary")),
+		Username: username,
+		Password: password,
+	}
+	db.WithConnection(ctx, n3Opts, t, func(conn *pgx.Conn) {
+		var version string
+		err := conn.QueryRow(ctx, "SELECT spock.spock_version();").Scan(&version)
+		require.NoError(t, err)
+		assert.Regexp(t, `^6\.`, version, "expected node n3 to be running Spock 6, got %q", version)
+	})
+}

server/internal/database/peer_catchup_resource.go

@@ -77,14 +77,20 @@ func (r *PeerCatchupResource) Refresh(ctx context.Context, rc *resource.Context)
 	}
 	defer conn.Close(ctx)
 
+	spockVersion, err := getLiveSpockVersion(ctx, conn)
+	if err != nil {
+		return fmt.Errorf("failed to check spock version on source node %q: %w", r.SourceNode, err)
+	}
+	spockMajor, _ := spockVersion.Major()
+
 	const pollInterval = 500 * time.Millisecond
 
 	for {
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
 
-		reached, err := postgres.SpockProgressReachedLSN(r.PeerNode, syncEvent.SyncEventLsn).
+		reached, err := postgres.SpockProgressReachedLSN(spockMajor, r.PeerNode, syncEvent.SyncEventLsn).
 			Scalar(ctx, conn)
 		if err != nil {
 			return fmt.Errorf("failed to query spock progress for peer %q: %w", r.PeerNode, err)

server/internal/database/reconcile_versions_test.go

@@ -122,6 +122,59 @@ func TestReconcileVersions(t *testing.T) {
 				},
 			},
 		},
+		{
+			// Before ds.ParseVersion accepted pre-release suffixes, this
+			// instance would be silently skipped at reconcile_versions.go's
+			// ds.ParsePgEdgeVersion call (line 141) and never appear in
+			// updatedInstances. It should now reconcile normally, with the
+			// pre-release suffix dropped by Normalize().
+			name: "spock beta version is not silently skipped",
+			spec: &database.StoredSpec{
+				Spec: &database.Spec{
+					PostgresVersion: "17.4",
+					SpockVersion:    "5",
+					Nodes: []*database.Node{
+						{Name: "n1", HostIDs: []string{"host-1"}},
+					},
+				},
+			},
+			instances: []*database.StoredInstance{
+				{
+					InstanceID:    "n1-host-1",
+					NodeName:      "n1",
+					HostID:        "host-1",
+					PgEdgeVersion: ds.MustParsePgEdgeVersion("17.4", "5"),
+				},
+			},
+			statuses: []*database.StoredInstanceStatus{
+				{
+					InstanceID: "n1-host-1",
+					Status: &database.InstanceStatus{
+						StatusUpdatedAt: utils.PointerTo(time.Now()),
+						Role:            utils.PointerTo(patroni.InstanceRolePrimary),
+						PostgresVersion: utils.PointerTo("17.5"),
+						SpockVersion:    utils.PointerTo("6.0.0-beta.1"),
+					},
+				},
+			},
+			expectedSpec: &database.StoredSpec{
+				Spec: &database.Spec{
+					PostgresVersion: "17.5",
+					SpockVersion:    "6",
+					Nodes: []*database.Node{
+						{Name: "n1", HostIDs: []string{"host-1"}},
+					},
+				},
+			},
+			expectedInstances: []*database.StoredInstance{
+				{
+					InstanceID:    "n1-host-1",
+					NodeName:      "n1",
+					HostID:        "host-1",
+					PgEdgeVersion: ds.MustParsePgEdgeVersion("17.5", "6"),
+				},
+			},
+		},
 		{
 			name: "all nodes updated spock only",
 			spec: &database.StoredSpec{

server/internal/database/sync_event_resource.go

@@ -13,18 +13,31 @@ import (
 
 var minSpockVersionForSyncEventArgs = ds.MustParseVersion(postgres.MinSpockVersionForSyncEventArgs)
 
+// getLiveSpockVersion queries conn directly for the Spock version actually
+// running on this connection, rather than trusting spec/stored state — spec
+// state can lag behind what's really deployed (e.g. mid add-node, mid
+// upgrade), and every SQL-shape decision gated on Spock version needs to
+// match what's really there.
+func getLiveSpockVersion(ctx context.Context, conn *pgx.Conn) (*ds.Version, error) {
+	versionStr, err := postgres.GetSpockVersion().Scalar(ctx, conn)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get spock version: %w", err)
+	}
+	version, err := ds.ParseVersion(versionStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse spock version %q: %w", versionStr, err)
+	}
+	return version, nil
+}
+
 // spockSupportsSyncEventArgs reports whether conn's Spock version is new
 // enough for spock.sync_event(boolean) and the 5-arg
 // spock.wait_for_sync_event(..., wait_if_disabled) — see
 // postgres.MinSpockVersionForSyncEventArgs.
 func spockSupportsSyncEventArgs(ctx context.Context, conn *pgx.Conn) (bool, error) {
-	versionStr, err := postgres.GetSpockVersion().Scalar(ctx, conn)
-	if err != nil {
-		return false, fmt.Errorf("failed to get spock version: %w", err)
-	}
-	version, err := ds.ParseVersion(versionStr)
+	version, err := getLiveSpockVersion(ctx, conn)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse spock version %q: %w", versionStr, err)
+		return false, err
 	}
 	return version.Compare(minSpockVersionForSyncEventArgs) >= 0, nil
 }