Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,15 @@ Key mechanisms (all under `pkg/sandbox/`):
(with SNI/Host-inspecting TCP firewall for domain allow/deny lists). Slots are pooled and
reused; slot indexes are allocated locally against the node's netns state (leftover
namespaces from a previous run are torn down by startup reclaim).
**Guest IP stack**: guest networking is **IPv4-only by default**. The host tap interface
currently has no IPv6 router configured (no router advertisements, no routable prefix), so
guests boot with `ipv6.disable=1` — the entire AF_INET6 stack is off. This eliminates the
~250 ms Happy Eyeballs penalty (RFC 8305) that occurs when a dual-stack kernel prefers AAAA
records and then times out on the unroutable `fe80::` link-local address produced by SLAAC.
The switch is `Slot.HasIPv6Router()` → `ProcessOptions.IPv6RouterConfigured` → the
`ipv6.disable` kernel cmdline arg; when the host networking layer gains a full IPv6 stack
(router + prefix per slot), flipping `HasIPv6Router()` to `true` enables dual-stack guests
without further code changes.
- **Sandbox proxy** (:5007, `pkg/proxy/`): reverse-proxies incoming traffic from client-proxy to
the sandbox's slot IP and requested port, enforcing per-sandbox traffic access tokens.
- Writes sandbox lifecycle **events** and cgroup **host stats** to ClickHouse; exports metrics via
Expand Down
18 changes: 14 additions & 4 deletions packages/orchestrator/pkg/sandbox/fc/kernel_args.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,20 @@ func buildKernelArgs(ipv4 string, options ProcessOptions) KernelArgs {
// Define kernel init path
"init": options.InitScriptPath,

// Networking IPv4 and IPv6
"ip": ipv4,
"ipv6.disable": "0",
"ipv6.autoconf": "1",
// Networking. The guest's IPv6 behaviour is determined by whether the host
// tap interface has an IPv6 router configured (see Slot.HasIPv6Router and
// ProcessOptions.IPv6RouterConfigured). Without a router, SLAAC produces
// only an unroutable fe80:: link-local address; leaving IPv6 enabled in
// that state causes the kernel to prefer AAAA records and attempt IPv6
// first on every outbound connection, adding a ~250 ms Happy Eyeballs
// penalty (RFC 8305) before falling back to IPv4.
"ip": ipv4,
"ipv6.disable": func() string {
if options.IPv6RouterConfigured {
return "0"
}
return "1"
Comment thread
AdaAibaby marked this conversation as resolved.
Comment thread
AdaAibaby marked this conversation as resolved.
}(),

// Wait 1 second before exiting FC after panic or reboot
"panic": "1",
Expand Down
18 changes: 14 additions & 4 deletions packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@ func TestBuildKernelArgs_DefaultIsUnchanged(t *testing.T) {
name: "production defaults",
options: ProcessOptions{InitScriptPath: "/sbin/init"},
want: "i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 +
" ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" +
" ipv6.disable=1 loglevel=1 panic=1 pci=off quiet" +
" random.trust_cpu=on reboot=k rootflags=discard",
},
{
name: "kvm clock",
options: ProcessOptions{InitScriptPath: "/sbin/init", KvmClock: true},
want: "clocksource=kvm-clock i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 +
" ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" +
" ipv6.disable=1 loglevel=1 panic=1 pci=off quiet" +
" random.trust_cpu=on reboot=k rootflags=discard",
},
{
Expand All @@ -43,17 +43,27 @@ func TestBuildKernelArgs_DefaultIsUnchanged(t *testing.T) {
name: "kernel logs",
options: ProcessOptions{InitScriptPath: "/sbin/init", KernelLogs: true},
want: "console=ttyS0 i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 +
" ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" +
" ipv6.disable=1 loglevel=5 panic=1 pci=off" +
" random.trust_cpu=on reboot=k rootflags=discard",
},
{
name: "systemd to kernel logs",
options: ProcessOptions{InitScriptPath: "/sbin/init", SystemdToKernelLogs: true},
want: "console=ttyS0 i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 +
" ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" +
" ipv6.disable=1 loglevel=5 panic=1 pci=off" +
" random.trust_cpu=on reboot=k rootflags=discard" +
" systemd.journald.forward_to_console",
},
{
// When the host tap has a full IPv6 router configured, IPv6 must be
// enabled in the guest (ipv6.disable=0) so SLAAC can obtain a routable
// address and dual-stack connections work without Happy Eyeballs penalty.
name: "ipv6 router configured",
options: ProcessOptions{InitScriptPath: "/sbin/init", IPv6RouterConfigured: true},
want: "i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 +
" ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" +
" random.trust_cpu=on reboot=k rootflags=discard",
},
}

for _, tt := range tests {
Expand Down
9 changes: 9 additions & 0 deletions packages/orchestrator/pkg/sandbox/fc/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,14 @@ type ProcessOptions struct {
// filesystem-only snapshot. A memory resume never re-reads the command line.
CmdlineArgs map[string]string

// IPv6RouterConfigured reports whether the host-side tap interface for this
// slot has an IPv6 router configured (router advertisements + routable prefix).
// When false (the default) the guest kernel boots with ipv6.disable=1 so that
// the unroutable fe80:: link-local address produced by SLAAC does not trigger
// the Happy Eyeballs ~250 ms fallback on every dual-stack outbound connection.
// Set from Slot.HasIPv6Router() at boot time.
IPv6RouterConfigured bool

// AccessToken, when non-nil, makes Create write the guest MMDS metadata
// (sandbox/template IDs, logs address, and the access-token hash) before the
// VM boots, so a cold-booted envd can authenticate /init the same way it does
Expand Down Expand Up @@ -376,6 +384,7 @@ func (p *Process) Create(

// IPv4 configuration - format: [local_ip]::[gateway_ip]:[netmask]:hostname:iface:dhcp_option:[dns]
ipv4 := fmt.Sprintf("%s::%s:%s:instance:%s:off:%s", p.slot.NamespaceIP(), p.slot.TapIPString(), p.slot.TapMaskString(), p.slot.VpeerName(), p.slot.TapName())
options.IPv6RouterConfigured = p.slot.HasIPv6Router()
kernelArgs := buildKernelArgs(ipv4, options).String()
err = p.client.setBootSource(ctx, kernelArgs, p.kernelPath)
if err != nil {
Expand Down
15 changes: 15 additions & 0 deletions packages/orchestrator/pkg/sandbox/network/slot.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,21 @@ func (s *Slot) TapMAC() string {
return tapMAC
}

// HasIPv6Router reports whether the host-side tap interface for this slot has
// an IPv6 router configured (i.e. sends router advertisements so the guest can
// obtain a routable IPv6 address via SLAAC).
//
// Today the host networking stack is IPv4-only — no RA is sent on the tap, so
// SLAAC produces only an unroutable fe80:: link-local address inside the guest.
// Returning false here causes the orchestrator to boot guests with
// ipv6.disable=1, eliminating the ~250 ms Happy Eyeballs penalty that arises
// when the guest tries IPv6 first and times out. When a full IPv6 stack is
// added to the host networking layer, flip this to true (or derive it from the
// slot's actual IPv6 configuration).
func (s *Slot) HasIPv6Router() bool {
return false
}

func (s *Slot) InitializeFirewall() error {
if s.Firewall != nil {
return fmt.Errorf("firewall is already initialized for slot %s", s.Key)
Expand Down