From be75159c0350307e288e06b7e42c4e01c4cf3581 Mon Sep 17 00:00:00 2001 From: shaolila Date: Tue, 18 Aug 2026 14:24:42 +0800 Subject: [PATCH 1/3] fix(fc): disable IPv6 in guest kernel to eliminate Happy Eyeballs latency The tap interface is configured IPv4-only: process.go builds the ip= kernel parameter with NamespaceIP/TapIP/TapMask but no IPv6 router or prefix is advertised on the host side of the tap. With ipv6.disable=0 and ipv6.autoconf=1 the guest kernel enables SLAAC, which at best yields an unroutable fe80:: link-local address and at worst waits for an RA that never arrives. The practical consequence is a ~250 ms Happy Eyeballs penalty (RFC 8305) on every outbound connection to a dual-stack host: the kernel prefers the AAAA address, the attempt fails with EHOSTUNREACH or a silent NDP timeout, then falls back to IPv4. pip install, npm install, and any HTTP API call to a major service (OpenAI, GitHub, PyPI, GCP) all have AAAA records, so every sandbox pays this cost on every first TCP connection per destination. Fix: set ipv6.disable=1 and drop the now-redundant ipv6.autoconf=1. This is a one-line kernel cmdline change with no other code impact. Full IPv6 support can be re-enabled once the host networking layer gains a complete stack (RA, prefix delegation, ip6tables). Fixes: https://github.com/e2b-dev/infra/issues/3585 --- packages/orchestrator/pkg/sandbox/fc/kernel_args.go | 13 +++++++++---- .../orchestrator/pkg/sandbox/fc/kernel_args_test.go | 8 ++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/fc/kernel_args.go b/packages/orchestrator/pkg/sandbox/fc/kernel_args.go index 4eab95ca78..44b8a355c6 100644 --- a/packages/orchestrator/pkg/sandbox/fc/kernel_args.go +++ b/packages/orchestrator/pkg/sandbox/fc/kernel_args.go @@ -91,10 +91,15 @@ func buildKernelArgs(ipv4 string, options ProcessOptions) KernelArgs { // Define kernel init path "init": options.InitScriptPath, - // Networking IPv4 and IPv6 - "ip": ipv4, - "ipv6.disable": "0", - "ipv6.autoconf": "1", + // Networking — IPv4 only. The tap interface is not configured with an + // IPv6 router or prefix, so SLAAC produces only an unroutable fe80:: + // link-local address. Leaving IPv6 enabled causes the kernel's address + // selection to prefer AAAA records and attempt IPv6 first on every + // outbound connection, adding a ~250 ms Happy Eyeballs penalty before + // falling back to IPv4. Disable it entirely until the host networking + // layer gains a complete IPv6 stack. See: https://github.com/e2b-dev/infra/issues/3585 + "ip": ipv4, + "ipv6.disable": "1", // Wait 1 second before exiting FC after panic or reboot "panic": "1", diff --git a/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go b/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go index 1003088831..483971ab8b 100644 --- a/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go +++ b/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go @@ -27,14 +27,14 @@ func TestBuildKernelArgs_DefaultIsUnchanged(t *testing.T) { name: "production defaults", options: ProcessOptions{InitScriptPath: "/sbin/init"}, want: "i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + + " ipv6.disable=1 loglevel=1 panic=1 pci=off quiet" + " random.trust_cpu=on reboot=k rootflags=discard", }, { name: "kvm clock", options: ProcessOptions{InitScriptPath: "/sbin/init", KvmClock: true}, want: "clocksource=kvm-clock i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + + " ipv6.disable=1 loglevel=1 panic=1 pci=off quiet" + " random.trust_cpu=on reboot=k rootflags=discard", }, { @@ -43,14 +43,14 @@ func TestBuildKernelArgs_DefaultIsUnchanged(t *testing.T) { name: "kernel logs", options: ProcessOptions{InitScriptPath: "/sbin/init", KernelLogs: true}, want: "console=ttyS0 i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" + + " ipv6.disable=1 loglevel=5 panic=1 pci=off" + " random.trust_cpu=on reboot=k rootflags=discard", }, { name: "systemd to kernel logs", options: ProcessOptions{InitScriptPath: "/sbin/init", SystemdToKernelLogs: true}, want: "console=ttyS0 i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" + + " ipv6.disable=1 loglevel=5 panic=1 pci=off" + " random.trust_cpu=on reboot=k rootflags=discard" + " systemd.journald.forward_to_console", }, From 926e6ed210d659bc16c2c92675d357019291bec2 Mon Sep 17 00:00:00 2001 From: shaolila Date: Tue, 18 Aug 2026 15:17:07 +0800 Subject: [PATCH 2/3] refactor(fc): derive ipv6.disable from slot's IPv6 router state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the guest kernel always booted with ipv6.disable=1 (hardcoded in kernel_args.go). This fix keeps the same safe default while making the flag slot-driven, so the codebase is ready for a future host-side IPv6 stack without a scattered search-and-replace. Changes: - Slot.HasIPv6Router() bool — the single source of truth for whether the host tap has an IPv6 router configured. Returns false today (no RA on the tap); flip to true (or derive from real IPv6 state) when the host gains IPv6. - ProcessOptions.IPv6RouterConfigured bool — carries the slot's answer into buildKernelArgs without coupling kernel_args.go to the network package. Set from Slot.HasIPv6Router() in Process.Create, one line before the call. - buildKernelArgs selects ipv6.disable=0 or =1 based on the flag instead of a string literal. The default (false) preserves the existing behaviour: guests without an IPv6 router on the host tap boot with ipv6.disable=1. - kernel_args_test.go adds an "ipv6 router configured" case that asserts ipv6.disable=0 when IPv6RouterConfigured=true. Fixes #3585. --- .../pkg/sandbox/fc/kernel_args.go | 23 +++++++++++-------- .../pkg/sandbox/fc/kernel_args_test.go | 10 ++++++++ .../orchestrator/pkg/sandbox/fc/process.go | 9 ++++++++ .../orchestrator/pkg/sandbox/network/slot.go | 15 ++++++++++++ 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/fc/kernel_args.go b/packages/orchestrator/pkg/sandbox/fc/kernel_args.go index 44b8a355c6..3835c3ee3e 100644 --- a/packages/orchestrator/pkg/sandbox/fc/kernel_args.go +++ b/packages/orchestrator/pkg/sandbox/fc/kernel_args.go @@ -91,15 +91,20 @@ func buildKernelArgs(ipv4 string, options ProcessOptions) KernelArgs { // Define kernel init path "init": options.InitScriptPath, - // Networking — IPv4 only. The tap interface is not configured with an - // IPv6 router or prefix, so SLAAC produces only an unroutable fe80:: - // link-local address. Leaving IPv6 enabled causes the kernel's address - // selection to prefer AAAA records and attempt IPv6 first on every - // outbound connection, adding a ~250 ms Happy Eyeballs penalty before - // falling back to IPv4. Disable it entirely until the host networking - // layer gains a complete IPv6 stack. See: https://github.com/e2b-dev/infra/issues/3585 - "ip": ipv4, - "ipv6.disable": "1", + // Networking. The guest's IPv6 behaviour is determined by whether the host + // tap interface has an IPv6 router configured (see Slot.HasIPv6Router and + // ProcessOptions.IPv6RouterConfigured). Without a router, SLAAC produces + // only an unroutable fe80:: link-local address; leaving IPv6 enabled in + // that state causes the kernel to prefer AAAA records and attempt IPv6 + // first on every outbound connection, adding a ~250 ms Happy Eyeballs + // penalty (RFC 8305) before falling back to IPv4. + "ip": ipv4, + "ipv6.disable": func() string { + if options.IPv6RouterConfigured { + return "0" + } + return "1" + }(), // Wait 1 second before exiting FC after panic or reboot "panic": "1", diff --git a/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go b/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go index 483971ab8b..3ac2313de6 100644 --- a/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go +++ b/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go @@ -54,6 +54,16 @@ func TestBuildKernelArgs_DefaultIsUnchanged(t *testing.T) { " random.trust_cpu=on reboot=k rootflags=discard" + " systemd.journald.forward_to_console", }, + { + // When the host tap has a full IPv6 router configured, IPv6 must be + // enabled in the guest (ipv6.disable=0) so SLAAC can obtain a routable + // address and dual-stack connections work without Happy Eyeballs penalty. + name: "ipv6 router configured", + options: ProcessOptions{InitScriptPath: "/sbin/init", IPv6RouterConfigured: true}, + want: "i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + + " ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + + " random.trust_cpu=on reboot=k rootflags=discard", + }, } for _, tt := range tests { diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index 365bd15a4e..2aca358938 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -112,6 +112,14 @@ type ProcessOptions struct { // filesystem-only snapshot. A memory resume never re-reads the command line. CmdlineArgs map[string]string + // IPv6RouterConfigured reports whether the host-side tap interface for this + // slot has an IPv6 router configured (router advertisements + routable prefix). + // When false (the default) the guest kernel boots with ipv6.disable=1 so that + // the unroutable fe80:: link-local address produced by SLAAC does not trigger + // the Happy Eyeballs ~250 ms fallback on every dual-stack outbound connection. + // Set from Slot.HasIPv6Router() at boot time. + IPv6RouterConfigured bool + // AccessToken, when non-nil, makes Create write the guest MMDS metadata // (sandbox/template IDs, logs address, and the access-token hash) before the // VM boots, so a cold-booted envd can authenticate /init the same way it does @@ -376,6 +384,7 @@ func (p *Process) Create( // IPv4 configuration - format: [local_ip]::[gateway_ip]:[netmask]:hostname:iface:dhcp_option:[dns] ipv4 := fmt.Sprintf("%s::%s:%s:instance:%s:off:%s", p.slot.NamespaceIP(), p.slot.TapIPString(), p.slot.TapMaskString(), p.slot.VpeerName(), p.slot.TapName()) + options.IPv6RouterConfigured = p.slot.HasIPv6Router() kernelArgs := buildKernelArgs(ipv4, options).String() err = p.client.setBootSource(ctx, kernelArgs, p.kernelPath) if err != nil { diff --git a/packages/orchestrator/pkg/sandbox/network/slot.go b/packages/orchestrator/pkg/sandbox/network/slot.go index 6539d73856..0ecf365196 100644 --- a/packages/orchestrator/pkg/sandbox/network/slot.go +++ b/packages/orchestrator/pkg/sandbox/network/slot.go @@ -230,6 +230,21 @@ func (s *Slot) TapMAC() string { return tapMAC } +// HasIPv6Router reports whether the host-side tap interface for this slot has +// an IPv6 router configured (i.e. sends router advertisements so the guest can +// obtain a routable IPv6 address via SLAAC). +// +// Today the host networking stack is IPv4-only — no RA is sent on the tap, so +// SLAAC produces only an unroutable fe80:: link-local address inside the guest. +// Returning false here causes the orchestrator to boot guests with +// ipv6.disable=1, eliminating the ~250 ms Happy Eyeballs penalty that arises +// when the guest tries IPv6 first and times out. When a full IPv6 stack is +// added to the host networking layer, flip this to true (or derive it from the +// slot's actual IPv6 configuration). +func (s *Slot) HasIPv6Router() bool { + return false +} + func (s *Slot) InitializeFirewall() error { if s.Firewall != nil { return fmt.Errorf("firewall is already initialized for slot %s", s.Key) From 6a91876d053893aca5886d9db8c7a798023b0bd9 Mon Sep 17 00:00:00 2001 From: shaolila Date: Tue, 18 Aug 2026 15:29:05 +0800 Subject: [PATCH 3/3] docs: document IPv4-only guest networking and dual-stack upgrade path --- docs/ARCHITECTURE.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 53a9856f13..dfdec5d99c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -160,6 +160,15 @@ Key mechanisms (all under `pkg/sandbox/`): (with SNI/Host-inspecting TCP firewall for domain allow/deny lists). Slots are pooled and reused; slot indexes are allocated locally against the node's netns state (leftover namespaces from a previous run are torn down by startup reclaim). + **Guest IP stack**: guest networking is **IPv4-only by default**. The host tap interface + currently has no IPv6 router configured (no router advertisements, no routable prefix), so + guests boot with `ipv6.disable=1` — the entire AF_INET6 stack is off. This eliminates the + ~250 ms Happy Eyeballs penalty (RFC 8305) that occurs when a dual-stack kernel prefers AAAA + records and then times out on the unroutable `fe80::` link-local address produced by SLAAC. + The switch is `Slot.HasIPv6Router()` → `ProcessOptions.IPv6RouterConfigured` → the + `ipv6.disable` kernel cmdline arg; when the host networking layer gains a full IPv6 stack + (router + prefix per slot), flipping `HasIPv6Router()` to `true` enables dual-stack guests + without further code changes. - **Sandbox proxy** (:5007, `pkg/proxy/`): reverse-proxies incoming traffic from client-proxy to the sandbox's slot IP and requested port, enforcing per-sandbox traffic access tokens. - Writes sandbox lifecycle **events** and cgroup **host stats** to ClickHouse; exports metrics via