redis-proxy: raise ElasticKV connection budget

README.md

@@ -202,9 +202,9 @@ docker run --rm \
   -listen :6479 \
   -primary redis.internal:6379 \
   -secondary elastickv.internal:6380 \
-  -elastickv-pool-size 4 \
-  -secondary-write-concurrency 2 \
-  -secondary-script-concurrency 1 \
+  -elastickv-pool-size 192 \
+  -secondary-write-concurrency 96 \
+  -secondary-script-concurrency 2 \
   -mode dual-write
 ```
 

adapter/redis.go

@@ -116,6 +116,10 @@ const (
 
 const (
 	redisDispatchTimeout = 10 * time.Second
+	// redisLuaDispatchTimeout gives EVAL/EVALSHA enough room for migration
+	// scripts that expand into thousands of Redis calls while keeping regular
+	// commands on the tighter dispatch deadline.
+	redisLuaDispatchTimeout = 30 * time.Second
 	// defaultRedisBlockWaitFallback is the safety-net poll interval for
 	// blocking-command wait loops when no in-process write signal arrives.
 	// Signals cover normal XADD / ZADD / ZINCRBY wakeups immediately; this

adapter/redis_lua.go

@@ -115,7 +115,7 @@ func (r *RedisServer) runLuaScript(conn redcon.Conn, script string, evalArgs [][
 		return
 	}
 
-	ctx, cancel := context.WithTimeout(r.handlerContext(), redisDispatchTimeout)
+	ctx, cancel := context.WithTimeout(r.handlerContext(), redisLuaDispatchTimeout)
 	defer cancel()
 
 	start := time.Now()
@@ -833,7 +833,7 @@ func (r *RedisServer) execLuaCompat(conn redcon.Conn, command string, args [][]b
 		stringArgs[i] = string(arg)
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), redisDispatchTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), redisLuaDispatchTimeout)
 	defer cancel()
 
 	var reply luaReply

adapter/redis_lua_context.go

@@ -3534,7 +3534,7 @@ func (c *luaScriptContext) commit() error {
 	}
 	sort.Strings(keys)
 
-	ctx, cancel := context.WithTimeout(c.scriptCtx(), redisDispatchTimeout)
+	ctx, cancel := context.WithTimeout(c.scriptCtx(), redisLuaDispatchTimeout)
 	defer cancel()
 
 	// Pre-allocate a commitTS so Delta key bytes can embed it before dispatch.

adapter/redis_peer_limiter.go

@@ -10,9 +10,10 @@ import (
 
 const (
 	redisPerPeerLimitEnv              = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS"
-	defaultRedisProxyPoolPeerCap      = 64
-	defaultRedisDedicatedPeerHeadroom = 64
-	defaultRedisPerPeerConnectionCap  = defaultRedisProxyPoolPeerCap + defaultRedisDedicatedPeerHeadroom
+	defaultRedisProxyPoolPeerCap      = 192
+	defaultRedisProxyReplicasPerPeer  = 2
+	defaultRedisDedicatedPeerHeadroom = 128
+	defaultRedisPerPeerConnectionCap  = defaultRedisProxyPoolPeerCap*defaultRedisProxyReplicasPerPeer + defaultRedisDedicatedPeerHeadroom
 	redisPeerLimitError               = "ERR max connections per client exceeded"
 	unknownRedisPeer                  = "unknown"
 )

adapter/redis_peer_limiter_test.go

@@ -15,7 +15,11 @@ func TestRedisPeerLimiterDefaultMatchesProxyPool(t *testing.T) {
 	t.Setenv(redisPerPeerLimitEnv, "")
 	limiter := newDefaultRedisPeerLimiter()
 	require.NotNil(t, limiter)
-	require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize+defaultRedisDedicatedPeerHeadroom, limiter.limit)
+	require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize, defaultRedisProxyPoolPeerCap)
+	require.Equal(t,
+		proxy.DefaultElasticKVBackendOptions().PoolSize*defaultRedisProxyReplicasPerPeer+defaultRedisDedicatedPeerHeadroom,
+		limiter.limit,
+	)
 }
 
 func TestRedisPeerLimiterRejectsAndReleases(t *testing.T) {

cmd/redis-proxy/main.go

@@ -19,9 +19,11 @@ import (
 )
 
 const (
-	sentryFlushTimeout          = 2 * time.Second
-	metricsShutdownTimeout      = 5 * time.Second
-	secondaryConcurrencyDivisor = 2
+	sentryFlushTimeout                = 2 * time.Second
+	metricsShutdownTimeout            = 5 * time.Second
+	secondaryWriteConcurrencyDivisor  = 2
+	secondaryScriptConcurrencyDivisor = 32
+	secondaryScriptConcurrencyCap     = 2
 )
 
 func main() {
@@ -53,7 +55,7 @@ func run() error {
 	flag.IntVar(&primaryPoolSize, "primary-pool-size", primaryPoolSize, "Primary Redis backend connection pool size")
 	flag.IntVar(&elasticKVPoolSize, "elastickv-pool-size", elasticKVPoolSize, "ElasticKV backend connection pool size")
 	flag.IntVar(&secondaryWriteConcurrency, "secondary-write-concurrency", secondaryWriteConcurrency, "Maximum concurrent asynchronous secondary writes including scripts (0 = half of secondary backend pool size)")
-	flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = half of secondary write concurrency)")
+	flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, capped at 2)")
 	flag.IntVar(&secondaryBlockingReplayConcurrency, "secondary-blocking-replay-concurrency", secondaryBlockingReplayConcurrency, "Maximum concurrent asynchronous secondary mutating blocking-command replays (0 = capped remaining secondary backend pool capacity after writes)")
 	flag.IntVar(&secondaryWriteQueueSize, "secondary-write-queue-size", secondaryWriteQueueSize, "Maximum queued asynchronous secondary writes (0 = derived from write concurrency)")
 	flag.IntVar(&secondaryScriptQueueSize, "secondary-script-queue-size", secondaryScriptQueueSize, "Maximum queued asynchronous secondary Lua-script writes (0 = derived from script concurrency)")
@@ -283,11 +285,15 @@ func secondaryBackendPoolSize(mode proxy.ProxyMode, primaryPoolSize, elasticKVPo
 }
 
 func defaultSecondaryWriteConcurrency(poolSize int) int {
-	return atLeastOne(poolSize / secondaryConcurrencyDivisor)
+	return atLeastOne(poolSize / secondaryWriteConcurrencyDivisor)
 }
 
 func defaultSecondaryScriptConcurrency(writeConcurrency int) int {
-	return atLeastOne(writeConcurrency / secondaryConcurrencyDivisor)
+	concurrency := atLeastOne(writeConcurrency / secondaryScriptConcurrencyDivisor)
+	if concurrency > secondaryScriptConcurrencyCap {
+		return secondaryScriptConcurrencyCap
+	}
+	return concurrency
 }
 
 func defaultSecondaryBlockingReplayConcurrency(poolSize, writeConcurrency int) int {

cmd/redis-proxy/main_test.go

@@ -74,16 +74,16 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			primaryPoolSize:         128,
 			elasticKVPoolSize:       64,
 			wantWriteConcurrency:    32,
-			wantScriptConcurrency:   16,
-			wantBlockingConcurrency: 20,
+			wantScriptConcurrency:   1,
+			wantBlockingConcurrency: 32,
 		},
 		{
 			name:                    "shadow mode derives from ElasticKV pool",
 			mode:                    proxy.ModeDualWriteShadow,
 			primaryPoolSize:         128,
 			elasticKVPoolSize:       8,
 			wantWriteConcurrency:    4,
-			wantScriptConcurrency:   2,
+			wantScriptConcurrency:   1,
 			wantBlockingConcurrency: 4,
 		},
 		{
@@ -92,8 +92,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			primaryPoolSize:         128,
 			elasticKVPoolSize:       4,
 			wantWriteConcurrency:    64,
-			wantScriptConcurrency:   32,
-			wantBlockingConcurrency: 20,
+			wantScriptConcurrency:   2,
+			wantBlockingConcurrency: 32,
 		},
 		{
 			name:                    "large remaining pool caps blocking replay",
@@ -102,8 +102,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			elasticKVPoolSize:       144,
 			writeConcurrency:        80,
 			wantWriteConcurrency:    80,
-			wantScriptConcurrency:   40,
-			wantBlockingConcurrency: 20,
+			wantScriptConcurrency:   2,
+			wantBlockingConcurrency: 32,
 		},
 		{
 			name:                    "explicit write keeps derived script",
@@ -112,7 +112,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			elasticKVPoolSize:       8,
 			writeConcurrency:        5,
 			wantWriteConcurrency:    5,
-			wantScriptConcurrency:   2,
+			wantScriptConcurrency:   1,
 			wantBlockingConcurrency: 3,
 		},
 		{
@@ -154,3 +154,28 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 		})
 	}
 }
+
+func TestDeriveSecondaryConcurrencyFromDefaultElasticKVPool(t *testing.T) {
+	poolSize := proxy.DefaultElasticKVBackendOptions().PoolSize
+	writeConcurrency, scriptConcurrency, blockingConcurrency := deriveSecondaryConcurrency(
+		proxy.ModeDualWrite,
+		proxy.DefaultBackendOptions().PoolSize,
+		poolSize,
+		0,
+		0,
+		0,
+	)
+
+	require.Equal(t, 192, poolSize)
+	require.Equal(t, 96, writeConcurrency)
+	require.Equal(t, 2, scriptConcurrency)
+	require.Equal(t, 32, blockingConcurrency)
+	require.NoError(t, validateSecondaryConcurrency(
+		proxy.ModeDualWrite,
+		proxy.DefaultBackendOptions().PoolSize,
+		poolSize,
+		writeConcurrency,
+		scriptConcurrency,
+		blockingConcurrency,
+	))
+}

deploy/redis-proxy/docker-compose.ha.yml

@@ -26,7 +26,7 @@ services:
       - -listen=:6379
       - -primary=${REDIS_PROXY_PRIMARY:-redis:6379}
       - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380}
-      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-64}
+      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-192}
       - -mode=${REDIS_PROXY_MODE:-dual-write-shadow}
       - -metrics=:9191
     networks:
@@ -46,7 +46,7 @@ services:
       - -listen=:6379
       - -primary=${REDIS_PROXY_PRIMARY:-redis:6379}
       - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380}
-      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-64}
+      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-192}
       - -mode=${REDIS_PROXY_MODE:-dual-write-shadow}
       - -metrics=:9191
     networks:

docs/redis-proxy-deployment.md

@@ -35,13 +35,15 @@ go build -o redis-proxy ./cmd/redis-proxy/
 | `-secondary-db` | `0` | Secondary Redis DB number |
 | `-secondary-password` | (empty) | Secondary Redis password |
 | `-primary-pool-size` | `128` | Primary Redis backend connection pool size |
-| `-elastickv-pool-size` | `64` | ElasticKV backend connection pool size |
+| `-elastickv-pool-size` | `192` | ElasticKV backend connection pool size |
 | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` |
-| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` |
+| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives one thirty-second of the shared write limit, capped at `2`, minimum `1` |
+| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `32` |
 | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` |
 | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` |
+| `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` |
 | `-mode` | `dual-write` | Proxy mode (see below) |
-| `-secondary-timeout` | `5s` | End-to-end secondary write deadline, including queue wait |
+| `-secondary-timeout` | `30s` | End-to-end secondary write deadline, including queue wait |
 | `-shadow-timeout` | `3s` | Shadow read timeout |
 | `-sentry-dsn` | (empty) | Sentry DSN (empty = disabled) |
 | `-sentry-env` | (empty) | Sentry environment name |
@@ -94,11 +96,11 @@ docker run --rm \
   -primary redis.internal:6379 \
   -primary-password "${REDIS_PASSWORD}" \
   -secondary elastickv.internal:6380 \
-  -elastickv-pool-size 64 \
-  -secondary-write-concurrency 32 \
-  -secondary-script-concurrency 16 \
+  -elastickv-pool-size 192 \
+  -secondary-write-concurrency 96 \
+  -secondary-script-concurrency 2 \
   -mode dual-write-shadow \
-  -secondary-timeout 5s \
+  -secondary-timeout 30s \
   -shadow-timeout 3s \
   -sentry-dsn "${SENTRY_DSN}" \
   -sentry-env production \
@@ -118,9 +120,9 @@ services:
       - -listen=:6479
       - -primary=redis:6379
       - -secondary=elastickv:6380
-      - -elastickv-pool-size=64
-      - -secondary-write-concurrency=32
-      - -secondary-script-concurrency=16
+      - -elastickv-pool-size=192
+      - -secondary-write-concurrency=96
+      - -secondary-script-concurrency=2
       - -mode=dual-write-shadow
       - -metrics=:9191
     depends_on:
@@ -213,7 +215,7 @@ Override backend wiring via env vars before `docker compose up`:
 ```bash
 REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \
 REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \
-REDIS_PROXY_ELASTICKV_POOL_SIZE=64 \
+REDIS_PROXY_ELASTICKV_POOL_SIZE=192 \
 REDIS_PROXY_MODE=dual-write-shadow \
   docker compose -f docker-compose.ha.yml up -d
 ```
@@ -414,7 +416,8 @@ groups:
 | Parameter | Value | Description |
 |-----------|-------|-------------|
 | Redis connection pool size | 128 | Default go-redis pool size for Redis |
-| ElasticKV connection pool size | 64 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections |
+| ElasticKV connection pool size | 192 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections |
+| ElasticKV Redis per-peer connection cap | 512 | Default server-side cap: two proxy replicas at pool `192`, plus dedicated PubSub/shadow PubSub headroom |
 | Dial timeout | 5s | Backend connection timeout |
 | Read timeout | 3s | Backend read timeout |
 | Write timeout | 3s | Backend write timeout |
@@ -439,7 +442,8 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`.
 ### Secondary writes are falling behind
 - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency.
 - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool.
-- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` above `-elastickv-pool-size`; PubSub and shadow PubSub use dedicated connections outside the command pool. Keep `-secondary-write-concurrency` at or below the pool size.
+- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `512`. Keep `-secondary-write-concurrency` at or below the pool size.
+- If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low and profile ElasticKV before raising it.
 - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency.
 
 ### High divergence count

proxy/backend.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	defaultPoolSize          = 128
-	defaultElasticKVPoolSize = 64
+	defaultElasticKVPoolSize = 192
 	defaultDialTimeout       = 5 * time.Second
 	defaultReadTimeout       = 3 * time.Second
 	defaultWriteTimeout      = 3 * time.Second
@@ -73,10 +73,11 @@ func DefaultBackendOptions() BackendOptions {
 
 // DefaultElasticKVBackendOptions returns defaults for proxy backends that
 // connect to ElasticKV's Redis adapter. Production dual-write deployments
-// should run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS above this
-// pool size because PubSub and shadow PubSub use dedicated connections outside
-// the go-redis command pool. Lower the proxy pool instead for clusters that
-// keep the server-side per-peer cap below the default.
+// should run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS sized for
+// every proxy replica that may share one client IP, plus dedicated PubSub and
+// shadow PubSub connections outside this command pool. Lower the proxy pool
+// instead for clusters that keep the server-side per-peer cap below the
+// default.
 func DefaultElasticKVBackendOptions() BackendOptions {
 	opts := DefaultBackendOptions()
 	opts.PoolSize = defaultElasticKVPoolSize

proxy/config.go

@@ -3,7 +3,7 @@ package proxy
 import "time"
 
 const (
-	defaultSecondaryTimeout    = 5 * time.Second
+	defaultSecondaryTimeout    = 30 * time.Second
 	defaultShadowTimeout       = 3 * time.Second
 	defaultPubSubCompareWindow = 2 * time.Second
 	defaultPubSubSweepInterval = 500 * time.Millisecond

proxy/dualwrite.go

@@ -33,7 +33,7 @@ const (
 	// from normal secondary writes. Blocking replays may wait for the secondary
 	// to observe a producer write, and they hit the secondary's heavy-command
 	// limiter, so keep the default well below the normal write limit.
-	maxBlockingReplayGoroutines = 20
+	maxBlockingReplayGoroutines = 32
 	// Async queues absorb short bursts without allowing an unavailable or slow
 	// secondary to build an unbounded replay backlog.
 	minAsyncQueueCapacity       = 64

proxy/proxy_test.go

@@ -183,6 +183,7 @@ func TestDefaultConfig(t *testing.T) {
 	assert.Equal(t, "localhost:6379", cfg.PrimaryAddr)
 	assert.Equal(t, "localhost:6380", cfg.SecondaryAddr)
 	assert.Equal(t, ModeDualWrite, cfg.Mode)
+	assert.Equal(t, 30*time.Second, cfg.SecondaryTimeout)
 }
 
 // ========== compare.go tests ==========
@@ -1334,7 +1335,7 @@ func TestDefaultBackendOptions(t *testing.T) {
 
 func TestDefaultElasticKVBackendOptions(t *testing.T) {
 	opts := DefaultElasticKVBackendOptions()
-	assert.Equal(t, 64, opts.PoolSize)
+	assert.Equal(t, 192, opts.PoolSize)
 	assert.Equal(t, 5*time.Second, opts.DialTimeout)
 	assert.Equal(t, 3*time.Second, opts.ReadTimeout)
 	assert.Equal(t, 3*time.Second, opts.WriteTimeout)