Tune redis proxy connection capacity

adapter/redis_peer_limiter.go

@@ -10,7 +10,7 @@ import (
 
 const (
 	redisPerPeerLimitEnv             = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS"
-	defaultRedisPerPeerConnectionCap = 8
+	defaultRedisPerPeerConnectionCap = 64
 	redisPeerLimitError              = "ERR max connections per client exceeded"
 	unknownRedisPeer                 = "unknown"
 )

adapter/redis_peer_limiter_test.go

@@ -78,7 +78,7 @@ func TestRedisLeaderClientPoolStaysBelowPeerLimit(t *testing.T) {
 }
 
 func TestRedisLeaderClientPoolUsesSmallDefault(t *testing.T) {
-	server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(8))
+	server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(defaultRedisPerPeerConnectionCap))
 	client := server.getOrCreateLeaderClient("127.0.0.1:6379")
 	defer client.Close()
 
@@ -95,7 +95,7 @@ func TestRedisLeaderClientPoolsSharePeerBudget(t *testing.T) {
 	}{
 		{name: "low cap", limit: 2, wantNormal: 1, wantBlocking: 1},
 		{name: "four cap", limit: 4, wantNormal: 2, wantBlocking: 2},
-		{name: "default cap", limit: 8, wantNormal: 4, wantBlocking: 4},
+		{name: "default cap", limit: defaultRedisPerPeerConnectionCap, wantNormal: defaultRedisLeaderClientPoolSize, wantBlocking: defaultRedisBlockingLeaderClientPoolSize},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(tc.limit))
@@ -107,7 +107,7 @@ func TestRedisLeaderClientPoolsSharePeerBudget(t *testing.T) {
 }
 
 func TestRedisBlockingLeaderClientUsesDedicatedBudgetedPool(t *testing.T) {
-	server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(8))
+	server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(defaultRedisPerPeerConnectionCap))
 	shared := server.getOrCreateLeaderClient("127.0.0.1:6379")
 	defer shared.Close()
 	blocking := server.getOrCreateBlockingLeaderClient("127.0.0.1:6379")

cmd/redis-proxy/main_test.go

@@ -72,10 +72,10 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			name:                    "dual write derives from ElasticKV pool",
 			mode:                    proxy.ModeDualWrite,
 			primaryPoolSize:         128,
-			elasticKVPoolSize:       4,
-			wantWriteConcurrency:    2,
-			wantScriptConcurrency:   1,
-			wantBlockingConcurrency: 2,
+			elasticKVPoolSize:       16,
+			wantWriteConcurrency:    8,
+			wantScriptConcurrency:   4,
+			wantBlockingConcurrency: 8,
 		},
 		{
 			name:                    "shadow mode derives from ElasticKV pool",

deploy/redis-proxy/docker-compose.ha.yml

@@ -26,7 +26,7 @@ services:
       - -listen=:6379
       - -primary=${REDIS_PROXY_PRIMARY:-redis:6379}
       - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380}
-      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-4}
+      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-16}
       - -mode=${REDIS_PROXY_MODE:-dual-write-shadow}
       - -metrics=:9191
     networks:
@@ -46,7 +46,7 @@ services:
       - -listen=:6379
       - -primary=${REDIS_PROXY_PRIMARY:-redis:6379}
       - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380}
-      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-4}
+      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-16}
       - -mode=${REDIS_PROXY_MODE:-dual-write-shadow}
       - -metrics=:9191
     networks:

docs/design/2026_04_24_implemented_workload_isolation.md

@@ -24,7 +24,7 @@ Implementation status:
   gated `EVAL`/`EVALSHA` path.
 - Shipped: Layer 3 per-peer Redis connection admission in
   `adapter/redis_peer_limiter.go`, wired through `RedisServer.Run` accept and
-  close hooks. Default cap is 8 per peer IP and is configurable via
+  close hooks. Default cap is 64 per peer IP and is configurable via
   `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` /
   `WithRedisPerPeerConnectionLimit`. Redis leader-proxy clients use a small
   explicit go-redis pool below that default cap, and Pub/Sub detached sockets
@@ -374,7 +374,7 @@ one check per accept, not per command.
 
 ### Recommended v1 shape
 
-**Per-peer-IP connection cap, default `N=8`, env-configurable,
+**Per-peer-IP connection cap, default `N=64`, env-configurable,
 enforced at accept.** On reject, accept the TCP connection, write a
 `-ERR max connections per client exceeded` RESP error, then close —
 so the client sees a protocol-level message instead of a bare

docs/redis-proxy-deployment.md

@@ -35,7 +35,7 @@ go build -o redis-proxy ./cmd/redis-proxy/
 | `-secondary-db` | `0` | Secondary Redis DB number |
 | `-secondary-password` | (empty) | Secondary Redis password |
 | `-primary-pool-size` | `128` | Primary Redis backend connection pool size |
-| `-elastickv-pool-size` | `4` | ElasticKV backend connection pool size |
+| `-elastickv-pool-size` | `16` | ElasticKV backend connection pool size |
 | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` |
 | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` |
 | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` |
@@ -94,9 +94,9 @@ docker run --rm \
   -primary redis.internal:6379 \
   -primary-password "${REDIS_PASSWORD}" \
   -secondary elastickv.internal:6380 \
-  -elastickv-pool-size 4 \
-  -secondary-write-concurrency 2 \
-  -secondary-script-concurrency 1 \
+  -elastickv-pool-size 16 \
+  -secondary-write-concurrency 8 \
+  -secondary-script-concurrency 4 \
   -mode dual-write-shadow \
   -secondary-timeout 5s \
   -shadow-timeout 3s \
@@ -118,9 +118,9 @@ services:
       - -listen=:6479
       - -primary=redis:6379
       - -secondary=elastickv:6380
-      - -elastickv-pool-size=4
-      - -secondary-write-concurrency=2
-      - -secondary-script-concurrency=1
+      - -elastickv-pool-size=16
+      - -secondary-write-concurrency=8
+      - -secondary-script-concurrency=4
       - -mode=dual-write-shadow
       - -metrics=:9191
     depends_on:
@@ -213,7 +213,7 @@ Override backend wiring via env vars before `docker compose up`:
 ```bash
 REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \
 REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \
-REDIS_PROXY_ELASTICKV_POOL_SIZE=4 \
+REDIS_PROXY_ELASTICKV_POOL_SIZE=16 \
 REDIS_PROXY_MODE=dual-write-shadow \
   docker compose -f docker-compose.ha.yml up -d
 ```
@@ -414,7 +414,7 @@ groups:
 | Parameter | Value | Description |
 |-----------|-------|-------------|
 | Redis connection pool size | 128 | Default go-redis pool size for Redis |
-| ElasticKV connection pool size | 4 | Default per-leader pool; keep within the server per-peer connection limit |
+| ElasticKV connection pool size | 16 | Default per-leader pool; keep within the server per-peer connection limit |
 | Dial timeout | 5s | Backend connection timeout |
 | Read timeout | 3s | Backend read timeout |
 | Write timeout | 3s | Backend write timeout |
@@ -439,7 +439,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`.
 ### Secondary writes are falling behind
 - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency.
 - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool.
-- Increase the ElasticKV pool only together with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS`; keep `-secondary-write-concurrency` at or below the pool size.
+- Increase the ElasticKV pool only together with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` when the proxy pool can exceed the server's per-peer cap; keep `-secondary-write-concurrency` at or below the pool size.
 - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency.
 
 ### High divergence count

proxy/backend.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	defaultPoolSize          = 128
-	defaultElasticKVPoolSize = 4
+	defaultElasticKVPoolSize = 16
 	defaultDialTimeout       = 5 * time.Second
 	defaultReadTimeout       = 3 * time.Second
 	defaultWriteTimeout      = 3 * time.Second

proxy/dualwrite.go

@@ -57,7 +57,7 @@ const (
 	// producer-before-consumer ordering settle; a bounded no-effect window keeps
 	// BZPOP load from turning into a long retry backlog.
 	blockingReplayInitialDelay        = 250 * time.Millisecond
-	blockingReplayNoEffectRetryWindow = 500 * time.Millisecond
+	blockingReplayNoEffectRetryWindow = 100 * time.Millisecond
 	// compactedRetryInitialBackoff is the first delay before retrying a secondary
 	// command that failed with a compacted-read error.
 	compactedRetryInitialBackoff = 10 * time.Millisecond

proxy/proxy_test.go

@@ -784,7 +784,7 @@ func TestDualWriter_Blocking_BZPopReplayShortTimeoutStillAttemptsZRem(t *testing
 
 func TestNoEffectReplayRetryLimitIncludesJitterBudget(t *testing.T) {
 	limit := noEffectReplayRetryLimit(context.Background(), blockingReplayNoEffectRetryWindow)
-	assert.Positive(t, limit)
+	assert.Equal(t, 2, limit)
 
 	var spent time.Duration
 	backoff := compactedRetryInitialBackoff
@@ -1334,7 +1334,7 @@ func TestDefaultBackendOptions(t *testing.T) {
 
 func TestDefaultElasticKVBackendOptions(t *testing.T) {
 	opts := DefaultElasticKVBackendOptions()
-	assert.Equal(t, 4, opts.PoolSize)
+	assert.Equal(t, 16, opts.PoolSize)
 	assert.Equal(t, 5*time.Second, opts.DialTimeout)
 	assert.Equal(t, 3*time.Second, opts.ReadTimeout)
 	assert.Equal(t, 3*time.Second, opts.WriteTimeout)