redis-proxy: raise ElasticKV connection budget

README.md

@@ -202,9 +202,9 @@ docker run --rm \
   -listen :6479 \
   -primary redis.internal:6379 \
   -secondary elastickv.internal:6380 \
-  -elastickv-pool-size 4 \
-  -secondary-write-concurrency 2 \
-  -secondary-script-concurrency 1 \
+  -elastickv-pool-size 192 \
+  -secondary-write-concurrency 96 \
+  -secondary-script-concurrency 3 \
   -mode dual-write
 ```
 

adapter/redis.go

@@ -116,6 +116,10 @@ const (
 
 const (
 	redisDispatchTimeout = 10 * time.Second
+	// redisLuaDispatchTimeout gives EVAL/EVALSHA enough room for migration
+	// scripts that expand into thousands of Redis calls while keeping regular
+	// commands on the tighter dispatch deadline.
+	redisLuaDispatchTimeout = 30 * time.Second
 	// defaultRedisBlockWaitFallback is the safety-net poll interval for
 	// blocking-command wait loops when no in-process write signal arrives.
 	// Signals cover normal XADD / ZADD / ZINCRBY wakeups immediately; this

adapter/redis_lua.go

@@ -115,7 +115,7 @@ func (r *RedisServer) runLuaScript(conn redcon.Conn, script string, evalArgs [][
 		return
 	}
 
-	ctx, cancel := context.WithTimeout(r.handlerContext(), redisDispatchTimeout)
+	ctx, cancel := context.WithTimeout(r.handlerContext(), redisLuaDispatchTimeout)
 	defer cancel()
 
 	start := time.Now()

adapter/redis_lua_context.go

@@ -3534,7 +3534,7 @@ func (c *luaScriptContext) commit() error {
 	}
 	sort.Strings(keys)
 
-	ctx, cancel := context.WithTimeout(c.scriptCtx(), redisDispatchTimeout)
+	ctx, cancel := context.WithTimeout(c.scriptCtx(), redisLuaDispatchTimeout)
 	defer cancel()
 
 	// Pre-allocate a commitTS so Delta key bytes can embed it before dispatch.

adapter/redis_peer_limiter.go

@@ -10,9 +10,10 @@ import (
 
 const (
 	redisPerPeerLimitEnv              = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS"
-	defaultRedisProxyPoolPeerCap      = 64
-	defaultRedisDedicatedPeerHeadroom = 64
-	defaultRedisPerPeerConnectionCap  = defaultRedisProxyPoolPeerCap + defaultRedisDedicatedPeerHeadroom
+	defaultRedisProxyPoolPeerCap      = 192
+	defaultRedisProxyReplicasPerPeer  = 2
+	defaultRedisDedicatedPeerHeadroom = 128
+	defaultRedisPerPeerConnectionCap  = defaultRedisProxyPoolPeerCap*defaultRedisProxyReplicasPerPeer + defaultRedisDedicatedPeerHeadroom
 	redisPeerLimitError               = "ERR max connections per client exceeded"
 	unknownRedisPeer                  = "unknown"
 )

adapter/redis_peer_limiter_test.go

@@ -15,7 +15,11 @@ func TestRedisPeerLimiterDefaultMatchesProxyPool(t *testing.T) {
 	t.Setenv(redisPerPeerLimitEnv, "")
 	limiter := newDefaultRedisPeerLimiter()
 	require.NotNil(t, limiter)
-	require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize+defaultRedisDedicatedPeerHeadroom, limiter.limit)
+	require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize, defaultRedisProxyPoolPeerCap)
+	require.Equal(t,
+		proxy.DefaultElasticKVBackendOptions().PoolSize*defaultRedisProxyReplicasPerPeer+defaultRedisDedicatedPeerHeadroom,
+		limiter.limit,
+	)
 }
 
 func TestRedisPeerLimiterRejectsAndReleases(t *testing.T) {

cmd/redis-proxy/main.go

@@ -19,9 +19,11 @@ import (
 )
 
 const (
-	sentryFlushTimeout          = 2 * time.Second
-	metricsShutdownTimeout      = 5 * time.Second
-	secondaryConcurrencyDivisor = 2
+	sentryFlushTimeout                = 2 * time.Second
+	metricsShutdownTimeout            = 5 * time.Second
+	secondaryWriteConcurrencyDivisor  = 2
+	secondaryScriptConcurrencyDivisor = 32
+	secondaryScriptConcurrencyCap     = 3
 )
 
 func main() {
@@ -53,13 +55,14 @@ func run() error {
 	flag.IntVar(&primaryPoolSize, "primary-pool-size", primaryPoolSize, "Primary Redis backend connection pool size")
 	flag.IntVar(&elasticKVPoolSize, "elastickv-pool-size", elasticKVPoolSize, "ElasticKV backend connection pool size")
 	flag.IntVar(&secondaryWriteConcurrency, "secondary-write-concurrency", secondaryWriteConcurrency, "Maximum concurrent asynchronous secondary writes including scripts (0 = half of secondary backend pool size)")
-	flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = half of secondary write concurrency)")
+	flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, minimum 1, capped at 3)")
 	flag.IntVar(&secondaryBlockingReplayConcurrency, "secondary-blocking-replay-concurrency", secondaryBlockingReplayConcurrency, "Maximum concurrent asynchronous secondary mutating blocking-command replays (0 = capped remaining secondary backend pool capacity after writes)")
 	flag.IntVar(&secondaryWriteQueueSize, "secondary-write-queue-size", secondaryWriteQueueSize, "Maximum queued asynchronous secondary writes (0 = derived from write concurrency)")
 	flag.IntVar(&secondaryScriptQueueSize, "secondary-script-queue-size", secondaryScriptQueueSize, "Maximum queued asynchronous secondary Lua-script writes (0 = derived from script concurrency)")
 	flag.IntVar(&secondaryBlockingReplayQueueSize, "secondary-blocking-replay-queue-size", secondaryBlockingReplayQueueSize, "Maximum queued asynchronous secondary mutating blocking-command replays (0 = derived from blocking replay concurrency)")
 	flag.StringVar(&modeStr, "mode", "dual-write", "Proxy mode: redis-only, dual-write, dual-write-shadow, elastickv-primary, elastickv-only")
 	flag.DurationVar(&cfg.SecondaryTimeout, "secondary-timeout", cfg.SecondaryTimeout, "Secondary write timeout")
+	flag.DurationVar(&cfg.SecondaryScriptTimeout, "secondary-script-timeout", cfg.SecondaryScriptTimeout, "Secondary Lua-script write timeout (0 = secondary-timeout)")
 	flag.DurationVar(&cfg.ShadowTimeout, "shadow-timeout", cfg.ShadowTimeout, "Shadow read timeout")
 	flag.StringVar(&cfg.SentryDSN, "sentry-dsn", cfg.SentryDSN, "Sentry DSN (empty = disabled)")
 	flag.StringVar(&cfg.SentryEnv, "sentry-env", cfg.SentryEnv, "Sentry environment")
@@ -283,11 +286,15 @@ func secondaryBackendPoolSize(mode proxy.ProxyMode, primaryPoolSize, elasticKVPo
 }
 
 func defaultSecondaryWriteConcurrency(poolSize int) int {
-	return atLeastOne(poolSize / secondaryConcurrencyDivisor)
+	return atLeastOne(poolSize / secondaryWriteConcurrencyDivisor)
 }
 
 func defaultSecondaryScriptConcurrency(writeConcurrency int) int {
-	return atLeastOne(writeConcurrency / secondaryConcurrencyDivisor)
+	concurrency := atLeastOne(writeConcurrency / secondaryScriptConcurrencyDivisor)
+	if concurrency > secondaryScriptConcurrencyCap {
+		return secondaryScriptConcurrencyCap
+	}
+	return concurrency
 }
 
 func defaultSecondaryBlockingReplayConcurrency(poolSize, writeConcurrency int) int {

cmd/redis-proxy/main_test.go

@@ -74,16 +74,16 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			primaryPoolSize:         128,
 			elasticKVPoolSize:       64,
 			wantWriteConcurrency:    32,
-			wantScriptConcurrency:   16,
-			wantBlockingConcurrency: 20,
+			wantScriptConcurrency:   1,
+			wantBlockingConcurrency: 32,
 		},
 		{
 			name:                    "shadow mode derives from ElasticKV pool",
 			mode:                    proxy.ModeDualWriteShadow,
 			primaryPoolSize:         128,
 			elasticKVPoolSize:       8,
 			wantWriteConcurrency:    4,
-			wantScriptConcurrency:   2,
+			wantScriptConcurrency:   1,
 			wantBlockingConcurrency: 4,
 		},
 		{
@@ -92,8 +92,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			primaryPoolSize:         128,
 			elasticKVPoolSize:       4,
 			wantWriteConcurrency:    64,
-			wantScriptConcurrency:   32,
-			wantBlockingConcurrency: 20,
+			wantScriptConcurrency:   2,
+			wantBlockingConcurrency: 32,
 		},
 		{
 			name:                    "large remaining pool caps blocking replay",
@@ -102,8 +102,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			elasticKVPoolSize:       144,
 			writeConcurrency:        80,
 			wantWriteConcurrency:    80,
-			wantScriptConcurrency:   40,
-			wantBlockingConcurrency: 20,
+			wantScriptConcurrency:   2,
+			wantBlockingConcurrency: 32,
 		},
 		{
 			name:                    "explicit write keeps derived script",
@@ -112,7 +112,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 			elasticKVPoolSize:       8,
 			writeConcurrency:        5,
 			wantWriteConcurrency:    5,
-			wantScriptConcurrency:   2,
+			wantScriptConcurrency:   1,
 			wantBlockingConcurrency: 3,
 		},
 		{
@@ -154,3 +154,28 @@ func TestDeriveSecondaryConcurrency(t *testing.T) {
 		})
 	}
 }
+
+func TestDeriveSecondaryConcurrencyFromDefaultElasticKVPool(t *testing.T) {
+	poolSize := proxy.DefaultElasticKVBackendOptions().PoolSize
+	writeConcurrency, scriptConcurrency, blockingConcurrency := deriveSecondaryConcurrency(
+		proxy.ModeDualWrite,
+		proxy.DefaultBackendOptions().PoolSize,
+		poolSize,
+		0,
+		0,
+		0,
+	)
+
+	require.Equal(t, 192, poolSize)
+	require.Equal(t, 96, writeConcurrency)
+	require.Equal(t, 3, scriptConcurrency)
+	require.Equal(t, 32, blockingConcurrency)
+	require.NoError(t, validateSecondaryConcurrency(
+		proxy.ModeDualWrite,
+		proxy.DefaultBackendOptions().PoolSize,
+		poolSize,
+		writeConcurrency,
+		scriptConcurrency,
+		blockingConcurrency,
+	))
+}

deploy/redis-proxy/docker-compose.ha.yml

@@ -26,7 +26,7 @@ services:
       - -listen=:6379
       - -primary=${REDIS_PROXY_PRIMARY:-redis:6379}
       - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380}
-      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-64}
+      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-192}
       - -mode=${REDIS_PROXY_MODE:-dual-write-shadow}
       - -metrics=:9191
     networks:
@@ -46,7 +46,7 @@ services:
       - -listen=:6379
       - -primary=${REDIS_PROXY_PRIMARY:-redis:6379}
       - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380}
-      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-64}
+      - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-192}
       - -mode=${REDIS_PROXY_MODE:-dual-write-shadow}
       - -metrics=:9191
     networks:

docs/architecture_overview.md

@@ -170,7 +170,7 @@ sequenceDiagram
   participant H as "HLC (all nodes)"
   participant Tx as "Txn / MVCC read-write path"
 
-  loop "every hlcRenewalInterval (<3s)"
+  loop "every hlcRenewalInterval (<10s)"
     L->>RG: "Propose HLC lease (now + hlcPhysicalWindowMs)"
     RG-->>F: "Apply HLC lease entry"
     F->>H: "SetPhysicalCeiling(ms)"

docs/redis-proxy-deployment.md

@@ -35,13 +35,16 @@ go build -o redis-proxy ./cmd/redis-proxy/
 | `-secondary-db` | `0` | Secondary Redis DB number |
 | `-secondary-password` | (empty) | Secondary Redis password |
 | `-primary-pool-size` | `128` | Primary Redis backend connection pool size |
-| `-elastickv-pool-size` | `64` | ElasticKV backend connection pool size |
+| `-elastickv-pool-size` | `192` | ElasticKV backend connection pool size |
 | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` |
-| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` |
+| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives one thirty-second of the shared write limit, capped at `3`, minimum `1` |
+| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `32` |
 | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` |
 | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` |
+| `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` |
 | `-mode` | `dual-write` | Proxy mode (see below) |
-| `-secondary-timeout` | `5s` | End-to-end secondary write deadline, including queue wait |
+| `-secondary-timeout` | `30s` | End-to-end secondary write deadline, including queue wait |
+| `-secondary-script-timeout` | `5m` | End-to-end secondary Lua-script write deadline, including queue wait. `0` follows `-secondary-timeout` |
 | `-shadow-timeout` | `3s` | Shadow read timeout |
 | `-sentry-dsn` | (empty) | Sentry DSN (empty = disabled) |
 | `-sentry-env` | (empty) | Sentry environment name |
@@ -94,11 +97,12 @@ docker run --rm \
   -primary redis.internal:6379 \
   -primary-password "${REDIS_PASSWORD}" \
   -secondary elastickv.internal:6380 \
-  -elastickv-pool-size 64 \
-  -secondary-write-concurrency 32 \
-  -secondary-script-concurrency 16 \
+  -elastickv-pool-size 192 \
+  -secondary-write-concurrency 96 \
+  -secondary-script-concurrency 3 \
   -mode dual-write-shadow \
-  -secondary-timeout 5s \
+  -secondary-timeout 30s \
+  -secondary-script-timeout 5m \
   -shadow-timeout 3s \
   -sentry-dsn "${SENTRY_DSN}" \
   -sentry-env production \
@@ -118,9 +122,9 @@ services:
       - -listen=:6479
       - -primary=redis:6379
       - -secondary=elastickv:6380
-      - -elastickv-pool-size=64
-      - -secondary-write-concurrency=32
-      - -secondary-script-concurrency=16
+      - -elastickv-pool-size=192
+      - -secondary-write-concurrency=96
+      - -secondary-script-concurrency=3
       - -mode=dual-write-shadow
       - -metrics=:9191
     depends_on:
@@ -213,7 +217,7 @@ Override backend wiring via env vars before `docker compose up`:
 ```bash
 REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \
 REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \
-REDIS_PROXY_ELASTICKV_POOL_SIZE=64 \
+REDIS_PROXY_ELASTICKV_POOL_SIZE=192 \
 REDIS_PROXY_MODE=dual-write-shadow \
   docker compose -f docker-compose.ha.yml up -d
 ```
@@ -414,9 +418,10 @@ groups:
 | Parameter | Value | Description |
 |-----------|-------|-------------|
 | Redis connection pool size | 128 | Default go-redis pool size for Redis |
-| ElasticKV connection pool size | 64 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections |
+| ElasticKV connection pool size | 192 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections |
+| ElasticKV Redis per-peer connection cap | 512 | Default server-side cap: two proxy replicas at pool `192`, plus dedicated PubSub/shadow PubSub headroom |
 | Dial timeout | 5s | Backend connection timeout |
-| Read timeout | 3s | Backend read timeout |
+| Read timeout | Redis: 3s, ElasticKV: 35s | Backend read timeout |
 | Write timeout | 3s | Backend write timeout |
 | Async write concurrency fallback | 4096 | Package fallback; the command derives a lower limit from backend pool size |
 | Shadow read goroutine limit | 1024 | Max concurrent shadow comparisons |
@@ -439,7 +444,8 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`.
 ### Secondary writes are falling behind
 - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency.
 - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool.
-- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` above `-elastickv-pool-size`; PubSub and shadow PubSub use dedicated connections outside the command pool. Keep `-secondary-write-concurrency` at or below the pool size.
+- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `512`. Keep `-secondary-write-concurrency` at or below the pool size.
+- If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low; use `-secondary-script-timeout` for burst backlog and profile ElasticKV before raising concurrency.
 - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency.
 
 ### High divergence count

kv/coordinator.go

@@ -36,16 +36,25 @@ const dispatchLeaderRetryBudget = 5 * time.Second
 const dispatchLeaderRetryInterval = 25 * time.Millisecond
 
 // hlcPhysicalWindowMs is the duration in milliseconds that the Raft-agreed
-// physical ceiling extends ahead of the current wall clock. Modelled after
-// TiDB's TSO 3-second window: the leader commits ceiling = now + window, and
-// renews before the window expires. A new leader inherits the committed ceiling
-// so it never issues timestamps that collide with the previous leader's window.
-const hlcPhysicalWindowMs int64 = 3_000
+// physical ceiling extends ahead of the current wall clock. The leader commits
+// ceiling = now + window, and renews before the window expires. A new leader
+// inherits the committed ceiling so it never issues timestamps that collide with
+// the previous leader's window. Keep this comfortably above the renewal proposal
+// timeout so a renewal that spends a few seconds queued in Raft still lands with
+// a future ceiling.
+const hlcPhysicalWindowMs int64 = 10_000
 
 // hlcRenewalInterval controls how often the leader proposes a new ceiling.
 // Must be less than hlcPhysicalWindowMs to guarantee the window never expires.
 const hlcRenewalInterval = 1 * time.Second
 
+// hlcRenewalProposalTimeout bounds a single HLC lease-renewal proposal. This is
+// intentionally longer than hlcRenewalInterval: under write-heavy Redis proxy
+// traffic, a renewal can sit behind normal Raft proposals for more than one
+// second, and timing it out there causes avoidable fail-closed timestamp
+// refusals.
+const hlcRenewalProposalTimeout = dispatchLeaderRetryBudget
+
 // CoordinatorOption is a functional option for Coordinate constructors.
 type CoordinatorOption func(*Coordinate)
 
@@ -855,10 +864,11 @@ func (c *Coordinate) extendLeaseAfterRenewal(dispatchStart monoclock.Instant, ex
 // RunHLCLeaseRenewal runs a background loop that periodically proposes a new
 // physical ceiling to the Raft cluster while this node is the leader.
 //
-// The ceiling is set to now + hlcPhysicalWindowMs (3 s) and is renewed every
+// The ceiling is set to now + hlcPhysicalWindowMs (10 s) and is renewed every
 // hlcRenewalInterval (1 s), mirroring TiDB's TSO window strategy. Because the
-// window is always at least 2 s ahead of any real timestamp, a new leader will
-// never issue timestamps that overlap with the previous leader's window.
+// window is always well ahead of any real timestamp during healthy renewal, a
+// new leader will never issue timestamps that overlap with the previous
+// leader's window.
 //
 // RunHLCLeaseRenewal blocks until ctx is cancelled; call it in a goroutine.
 func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) {

kv/sharded_coordinator.go

@@ -2359,7 +2359,7 @@ func (c *ShardedCoordinator) renewHLCLeases(ctx context.Context) <-chan struct{}
 		go func(gid uint64, group *ShardGroup) {
 			defer wg.Done()
 			defer c.finishHLCLeaseRenewal(gid)
-			pctx, cancel := context.WithTimeout(ctx, hlcRenewalInterval)
+			pctx, cancel := context.WithTimeout(ctx, hlcRenewalProposalTimeout)
 			defer cancel()
 			c.renewHLCLease(pctx, gid, group)
 		}(gid, group)