From 0aa9bf5b33f6ee8d5ab1dd2e90e9a360481181b4 Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Wed, 29 Jul 2026 15:47:38 -0700 Subject: [PATCH] Add RDMA sysfs topology construction for gVisor sandboxes Containers with /dev/infiniband/uverbs* devices need a sysfs surface for device discovery: libibverbs enumerates /sys/class/infiniband_verbs, and NCCL builds its hardware topology by resolving /sys/class symlinks with realpath() and walking the PCI hierarchy. This adds a --rdmaproxy flag which will gate all RDMA support. For now it only wires up the sysfs topology: runsc snapshots the host sysfs closure derived from the spec's uverbs devices (ibdevs, netdevs, PCI ancestor chains, GPU functions, NUMA nodes) before pivot_root, and the sentry constructs a path-identical virtual tree: one canonical /sys/devices/pci... subtree per device, with /sys/class/* and /sys/bus/pci/devices as symlink farms matching the kernel layout. Per-port dynamic state (RoCE GID tables, link state, counters) is served live through targeted read-only bind mounts of the per-ibdev ports subtrees; identity and PCI attributes are a static snapshot, since the collecting process and the constructed sandbox tree do not observe the same sysfs mount (the RDMA netdevs move into the sandbox network namespace only after the sandbox is created). Each PCI function directory also gets a subsystem symlink into a synthesized /sys/bus/pci, and every node's bus gets a pci_bus entry: NCCL classifies a directory as a PCI device by following its subsystem symlink, and aborts topology discovery without it. The verbs device proxy that makes RDMA usable end-to-end is not part of this change; --rdmaproxy is a work in progress and RDMA is not yet functional. Co-authored-by: Alessio Ricci Toniolo FUTURE_COPYBARA_INTEGRATE_REVIEW=https://github.com/google/gvisor/pull/13712 from ayushr2:rdma-sysfs 24b199bc9d6d85cb49ffdb7085f2a95c7a491f90 PiperOrigin-RevId: 956129599 --- pkg/rdma/BUILD | 20 + pkg/rdma/collect.go | 461 ++++++++++++++++++ pkg/rdma/snapshot.go | 156 ++++++ pkg/rdma/snapshot_test.go | 128 +++++ pkg/sentry/fsimpl/sys/BUILD | 3 + pkg/sentry/fsimpl/sys/rdma.go | 361 ++++++++++++++ pkg/sentry/fsimpl/sys/sys.go | 58 ++- pkg/sentry/fsimpl/sys/sys_integration_test.go | 165 ++++++- runsc/boot/BUILD | 1 + runsc/boot/loader.go | 10 + runsc/boot/vfs.go | 12 +- runsc/cmd/BUILD | 1 + runsc/cmd/boot.go | 11 + runsc/cmd/chroot.go | 94 +++- runsc/config/config.go | 6 + runsc/config/flags.go | 1 + runsc/specutils/BUILD | 2 + runsc/specutils/rdma.go | 63 +++ 18 files changed, 1531 insertions(+), 22 deletions(-) create mode 100644 pkg/rdma/BUILD create mode 100644 pkg/rdma/collect.go create mode 100644 pkg/rdma/snapshot.go create mode 100644 pkg/rdma/snapshot_test.go create mode 100644 pkg/sentry/fsimpl/sys/rdma.go create mode 100644 runsc/specutils/rdma.go diff --git a/pkg/rdma/BUILD b/pkg/rdma/BUILD new file mode 100644 index 00000000000..155f766abc0 --- /dev/null +++ b/pkg/rdma/BUILD @@ -0,0 +1,20 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +go_library( + name = "rdma", + srcs = [ + "collect.go", + "snapshot.go", + ], + visibility = ["//visibility:public"], +) + +go_test( + name = "rdma_test", + srcs = ["snapshot_test.go"], + library = ":rdma", +) diff --git a/pkg/rdma/collect.go b/pkg/rdma/collect.go new file mode 100644 index 00000000000..48971a1fe72 --- /dev/null +++ b/pkg/rdma/collect.go @@ -0,0 +1,461 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdma + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" +) + +// UverbsSpec identifies one /dev/infiniband/uverbs* device from the +// container's OCI spec. +type UverbsSpec struct { + // Name is the device file name, e.g. "uverbs0". + Name string + // Major and Minor are the char device numbers from the spec. + Major int64 + Minor int64 +} + +// Static PCI attribute files captured at every level of the hierarchy. +// uevent is captured verbatim (NCCL parses PCI_SLOT_NAME out of it; +// libibverbs matches drivers on modalias). +var pciAttrNames = []string{ + "class", "vendor", "device", "subsystem_vendor", "subsystem_device", + "revision", "numa_node", "local_cpus", "local_cpulist", + "max_link_speed", "max_link_width", "current_link_speed", + "current_link_width", "modalias", "uevent", +} + +// Static identity attributes of /sys/class/infiniband//. +var ibAttrNames = []string{ + "node_type", "node_guid", "sys_image_guid", "fw_ver", "board_id", + "hca_type", "hw_rev", "node_desc", +} + +// Per-port attributes served live (they change at runtime: the RoCE GID +// table repopulates when netdevs move namespaces and acquire addresses; +// link state and rate can change on retrain). +var portLiveAttrNames = []string{ + "state", "phys_state", "rate", "lid", "sm_lid", "sm_sl", +} + +// Per-port attributes that are fixed for the sandbox lifetime. +var portStaticAttrNames = []string{"link_layer", "cap_mask"} + +// Curated netdev attribute set (see NetDev for why these are static). +var netAttrNames = []string{ + "address", "addr_len", "type", "dev_id", "dev_port", "mtu", "speed", + "duplex", "operstate", "carrier", "ifindex", +} + +// NUMA aggregate range files. +var numaAggregateNames = []string{ + "online", "possible", "has_cpu", "has_memory", "has_normal_memory", +} + +// Per-NUMA-node attribute files. +var numaNodeAttrNames = []string{"cpumap", "cpulist", "distance"} + +// GPU/accelerator PCI classes included as leaves (beyond the NIC leaves +// derived from the spec): 3D controller, VGA controller, NVSwitch bridge. +var gpuClassPrefixes = []string{"0x0302", "0x0300", "0x0680"} + +// Collect builds a snapshot for the given spec devices by reading host +// sysfs rooted at sysRoot (normally "/sys"; overridable for tests). +// +// The snapshot is collected by runsc during sandbox creation (while host +// sysfs is still reachable, before pivot_root) and consumed by the sentry's +// sysfs implementation to build a virtual /sys tree that is path-identical +// to the host layout: one canonical /sys/devices/pci... subtree per device, +// with /sys/class/* and /sys/bus/pci/devices as pure symlink farms. Path +// identity matters because consumers (libibverbs, NCCL) resolve symlinks +// with realpath() and walk the resolved paths. +// +// Only devices derived from the container's OCI spec are captured: the +// closure starts at the spec's /dev/infiniband/uverbs* entries and expands +// to their ibdevs, associated netdevs, PCI ancestor chains, GPU PCI +// functions, and the NUMA node topology. Nothing outside that closure is +// exposed to the sandbox. +// +// It must run while the RDMA netdevs are still in the host netns. +func Collect(sysRoot string, uverbs []UverbsSpec) (*Snapshot, error) { + if len(uverbs) == 0 { + return nil, nil + } + abi, err := mustReadAttr(path.Join(sysRoot, "class/infiniband_verbs/abi_version")) + if err != nil { + return nil, fmt.Errorf("reading verbs abi_version: %w", err) + } + s := &Snapshot{VerbsABIVersion: abi} + + // pciPaths accumulates every PCI directory (relative to sysRoot) in + // the closure; expanded with ancestors below. + pciPaths := make(map[string]bool) + + for _, u := range uverbs { + if !SafeName(u.Name) { + return nil, fmt.Errorf("unsafe uverbs device name %q", u.Name) + } + uvDir := path.Join(sysRoot, "class/infiniband_verbs", u.Name) + // dev is reproduced verbatim as the uverbs "dev" file, but the + // major:minor check needs the value without its trailing newline. + dev, err := mustReadAttr(path.Join(uvDir, "dev")) + if err != nil { + return nil, fmt.Errorf("uverbs device %q: reading dev: %w", u.Name, err) + } + if want := fmt.Sprintf("%d:%d", u.Major, u.Minor); strings.TrimSpace(dev) != want { + return nil, fmt.Errorf("uverbs device %q: sysfs dev is %q but the "+ + "OCI spec says %q; the host may have renumbered devices since "+ + "the spec was built", u.Name, strings.TrimSpace(dev), want) + } + // ibdev is a directory name (used in paths and symlink targets), so + // keep the trimmed value; its file reproduction re-adds the newline. + ibdevRaw, err := mustReadAttr(path.Join(uvDir, "ibdev")) + if err != nil { + return nil, fmt.Errorf("uverbs device %q: reading ibdev: %w", u.Name, err) + } + ibdev := strings.TrimSpace(ibdevRaw) + if !SafeName(ibdev) { + return nil, fmt.Errorf("uverbs device %q: unusable ibdev name %q", u.Name, ibdev) + } + ibDir := path.Join(sysRoot, "class/infiniband", ibdev) + + leaf, err := relRealpath(sysRoot, path.Join(ibDir, "device")) + if err != nil { + return nil, fmt.Errorf("resolving PCI device of %s: %w", ibdev, err) + } + if err := addWithAncestors(pciPaths, leaf); err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + + abiVersion, err := mustReadAttr(path.Join(uvDir, "abi_version")) + if err != nil { + return nil, fmt.Errorf("uverbs device %q: reading abi_version: %w", u.Name, err) + } + ibAttrs, err := readAttrs(ibDir, ibAttrNames) + if err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + d := Device{ + Uverbs: u.Name, + IBDev: ibdev, + LeafPCI: leaf, + Dev: dev, + ABIVersion: abiVersion, + IBAttrs: ibAttrs, + Ports: map[string]Port{}, + } + if err := d.collectPorts(ibDir); err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + if err := d.collectNetDevs(sysRoot, path.Join(ibDir, "device/net")); err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + s.Devices = append(s.Devices, d) + } + + // GPU/accelerator leaves by PCI class scan; fold each into the PCI + // closure so its ancestors are materialized alongside the NIC leaves. + gpus, err := gpuLeaves(sysRoot) + if err != nil { + return nil, err + } + for _, g := range gpus { + if err := addWithAncestors(pciPaths, g); err != nil { + return nil, err + } + } + + // Materialize every PCI node with its static attributes. + for p := range pciPaths { + attrs, err := readAttrs(path.Join(sysRoot, p), pciAttrNames) + if err != nil { + return nil, fmt.Errorf("PCI node %q: %w", p, err) + } + s.PCINodes = append(s.PCINodes, PCINode{Path: p, Attrs: attrs}) + } + sort.Slice(s.PCINodes, func(i, j int) bool { return s.PCINodes[i].Path < s.PCINodes[j].Path }) + + numa, err := collectNUMA(sysRoot) + if err != nil { + return nil, err + } + s.NUMA = numa + return s, nil +} + +// addWithAncestors records leaf and every ancestor directory whose name is +// a PCI function or root complex. The walk is textual: no symlinks are +// followed (leaf is already fully resolved). +func addWithAncestors(set map[string]bool, leaf string) error { + if !strings.HasPrefix(leaf, "devices/pci") { + return fmt.Errorf("PCI path %q is not under devices/pci*", leaf) + } + p := leaf + for { + base := path.Base(p) + if bdfRE.MatchString(base) { + set[p] = true + p = path.Dir(p) + continue + } + if pciRootRE.MatchString(base) { + set[p] = true + return nil + } + return fmt.Errorf("unexpected component %q in PCI path %q", base, leaf) + } +} + +func (d *Device) collectPorts(ibDir string) error { + portsDir := path.Join(ibDir, "ports") + ents, err := os.ReadDir(portsDir) + if err != nil { + return fmt.Errorf("reading %q: %w", portsDir, err) + } + for _, e := range ents { + num := e.Name() + if !SafeName(num) { + continue + } + pDir := path.Join(portsDir, num) + staticAttrs, err := readAttrs(pDir, portStaticAttrNames) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + // gids is a mandatory part of every IB/RoCE port; its absence + // means a broken port tree, so any failure is fatal. counters and + // hw_counters vary by device, so a missing directory is tolerated. + gids, err := listSafeNames(path.Join(pDir, "gids")) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + counters, err := listOptionalNames(path.Join(pDir, "counters")) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + hwCounters, err := listOptionalNames(path.Join(pDir, "hw_counters")) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + d.Ports[num] = Port{ + StaticAttrs: staticAttrs, + LiveAttrs: append([]string{}, portLiveAttrNames...), + GIDNames: gids, + CounterNames: counters, + HWCounterNames: hwCounters, + } + } + if len(d.Ports) == 0 { + return fmt.Errorf("no ports under %q", portsDir) + } + return nil +} + +func (d *Device) collectNetDevs(sysRoot, netDir string) error { + ents, err := os.ReadDir(netDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // InfiniBand-link-layer devices may have no netdev; fine. + return nil + } + return fmt.Errorf("reading %q: %w", netDir, err) + } + for _, e := range ents { + name := e.Name() + if !SafeName(name) { + continue + } + attrs, err := readAttrs(path.Join(sysRoot, "class/net", name), netAttrNames) + if err != nil { + return err + } + d.NetDevs = append(d.NetDevs, NetDev{Name: name, Attrs: attrs}) + } + sort.Slice(d.NetDevs, func(i, j int) bool { return d.NetDevs[i].Name < d.NetDevs[j].Name }) + return nil +} + +// gpuLeaves returns the leaf paths (relative to sysRoot) of every host GPU, +// found by scanning /sys/bus/pci/devices for the GPU PCI classes. NCCL needs +// the GPU PCI positions to compute GPU<->NIC distance. We include all host +// GPUs: the exposure is read-only PCI metadata (NCCL discovers GPUs via CUDA, +// not sysfs, so extra nodes are inert). +func gpuLeaves(sysRoot string) ([]string, error) { + busDir := path.Join(sysRoot, "bus/pci/devices") + ents, err := os.ReadDir(busDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // No PCI bus exposed (e.g. tests); GPU-less snapshot. + return nil, nil + } + return nil, fmt.Errorf("reading %q: %w", busDir, err) + } + var leaves []string + for _, e := range ents { + // This scans unrelated host PCI devices, so a device that vanishes + // mid-scan (absent class) is skipped rather than failing collection; + // other read errors are still surfaced. + class, present, err := readAttr(path.Join(busDir, e.Name(), "class")) + if err != nil { + return nil, fmt.Errorf("reading class of PCI device %q: %w", e.Name(), err) + } + if !present { + continue + } + match := false + for _, p := range gpuClassPrefixes { + if strings.HasPrefix(strings.TrimSpace(class), p) { + match = true + break + } + } + if !match { + continue + } + leaf, err := relRealpath(sysRoot, path.Join(busDir, e.Name())) + if err != nil { + return nil, fmt.Errorf("resolving PCI path of GPU %q: %w", e.Name(), err) + } + leaves = append(leaves, leaf) + } + return leaves, nil +} + +func collectNUMA(sysRoot string) (*NUMA, error) { + nodeDir := path.Join(sysRoot, "devices/system/node") + ents, err := os.ReadDir(nodeDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Non-NUMA system (or tests): no node topology to expose. + return nil, nil + } + return nil, fmt.Errorf("reading %q: %w", nodeDir, err) + } + agg, err := readAttrs(nodeDir, numaAggregateNames) + if err != nil { + return nil, err + } + n := &NUMA{ + Aggregate: agg, + Nodes: map[string]map[string]string{}, + } + for _, e := range ents { + name := e.Name() + if !strings.HasPrefix(name, "node") || !SafeName(name) { + continue + } + attrs, err := readAttrs(path.Join(nodeDir, name), numaNodeAttrNames) + if err != nil { + return nil, err + } + n.Nodes[name] = attrs + } + return n, nil +} + +// relRealpath resolves p and returns it relative to sysRoot. +func relRealpath(sysRoot, p string) (string, error) { + r, err := filepath.EvalSymlinks(p) + if err != nil { + return "", err + } + rel, err := filepath.Rel(sysRoot, r) + if err != nil || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("%q resolves outside %q", p, sysRoot) + } + return rel, nil +} + +// readAttr reads the attribute file at p and returns its contents. present is +// false only when the file is absent (ENOENT). Any other error (permissions, +// I/O) is returned rather than swallowed. +func readAttr(p string) (content string, present bool, err error) { + b, err := os.ReadFile(p) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, nil + } + return "", false, err + } + return string(b), true, nil +} + +// mustReadAttr reads an attribute file that is expected to have. Unlike +// readAttr, it treats absence (ENOENT) as an error. +func mustReadAttr(p string) (string, error) { + content, present, err := readAttr(p) + if err != nil { + return "", err + } + if !present { + return "", fmt.Errorf("required sysfs file %q is missing", p) + } + return content, nil +} + +// readAttrs reads the named attribute files under dir. Absent files (ENOENT) +// are omitted so the sandbox mirrors the host's exact set of files. Any +// non-ENOENT read error aborts collection. +func readAttrs(dir string, names []string) (map[string]string, error) { + m := make(map[string]string) + for _, name := range names { + p := path.Join(dir, name) + content, present, err := readAttr(p) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", p, err) + } + if present { + m[name] = content + } + } + return m, nil +} + +// listSafeNames returns the safe entry names under dir, sorted. The ReadDir +// error is returned (never swallowed); callers decide whether an absent +// directory is acceptable (see listOptionalNames). +func listSafeNames(dir string) ([]string, error) { + ents, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", dir, err) + } + var names []string + for _, e := range ents { + if SafeName(e.Name()) { + names = append(names, e.Name()) + } + } + sort.Strings(names) + return names, nil +} + +// listOptionalNames is listSafeNames for a directory that may legitimately be +// absent: a missing directory yields (nil, nil), but a read error is still +// surfaced. +func listOptionalNames(dir string) ([]string, error) { + names, err := listSafeNames(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return names, err +} diff --git a/pkg/rdma/snapshot.go b/pkg/rdma/snapshot.go new file mode 100644 index 00000000000..3f2fa16d01a --- /dev/null +++ b/pkg/rdma/snapshot.go @@ -0,0 +1,156 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rdma hosts RDMA configuration and utilities shared between the +// sentry and runsc. +package rdma + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" +) + +// Path is the location inside the sandbox chroot where the serialized +// snapshot is stored. Needed to communicate this information across the boot +// process re-exec boundary. +const Path = "/var/lib/gvisor/rdma_sysfs.json" + +// PCINode is one directory in the /sys/devices/pci... hierarchy (a root +// complex, bridge, or leaf function) with its static attribute files. +type PCINode struct { + // Path is relative to /sys, e.g. + // "devices/pci0000:07/0000:07:01.0/0000:0c:00.0". + Path string `json:"path"` + // Attrs maps attribute file name to contents (verbatim, including any + // trailing newline). + Attrs map[string]string `json:"attrs"` +} + +// Port is the per-IB-port state. Attributes split into static (immutable +// for the sandbox lifetime, snapshotted) and live (served by reading the +// host file at access time through a bind mount; the RoCE GID table +// changes when netdevs move between namespaces and acquire addresses). +type Port struct { + StaticAttrs map[string]string `json:"static_attrs"` + LiveAttrs []string `json:"live_attrs"` + // GIDNames are the entry names of the GID table (typically "0".."255"). + // The same names index gids/, gid_attrs/types/ and + // gid_attrs/ndevs/, all served live. + GIDNames []string `json:"gid_names"` + CounterNames []string `json:"counter_names"` + HWCounterNames []string `json:"hw_counter_names"` +} + +// NetDev is a network device associated with an RDMA device, with its +// curated static attribute set. +type NetDev struct { + Name string `json:"name"` + Attrs map[string]string `json:"attrs"` +} + +// Device is one uverbs device and everything hanging off it. +type Device struct { + // Uverbs is the device name, e.g. "uverbs0". + Uverbs string `json:"uverbs"` + // IBDev is the InfiniBand device name, e.g. "mlx5_0". + IBDev string `json:"ibdev"` + // LeafPCI is the PCI function directory, relative to /sys. + LeafPCI string `json:"leaf_pci"` + // Dev is the host "major:minor" of the uverbs char device. + Dev string `json:"dev"` + ABIVersion string `json:"abi_version"` + // IBAttrs are the static identity attributes of + // /sys/class/infiniband// (node_guid, fw_ver, ...). + IBAttrs map[string]string `json:"ib_attrs"` + // Ports maps port number ("1") to its state. + Ports map[string]Port `json:"ports"` + // NetDevs are the netdevs bound to this ibdev via device/net/. + NetDevs []NetDev `json:"netdevs"` +} + +// NUMA is the /sys/devices/system/node subtree. +type NUMA struct { + // Aggregate holds the top-level range files (online, possible, ...). + Aggregate map[string]string `json:"aggregate"` + // Nodes maps node ID ("0") to its attribute files. + Nodes map[string]map[string]string `json:"nodes"` +} + +// Snapshot is the host sysfs snapshot for RDMA device topology. +type Snapshot struct { + VerbsABIVersion string `json:"verbs_abi_version"` + // PCINodes contains every PCI directory in the closure: the leaves + // (NICs, GPUs) and all their ancestor bridges/roots. Sorted by Path, + // which places parents before children. + PCINodes []PCINode `json:"pci_nodes"` + Devices []Device `json:"devices"` + NUMA *NUMA `json:"numa,omitempty"` +} + +// safeName matches names we are willing to reproduce inside the sandbox or +// join into host paths. Kernel-generated sysfs names satisfy this; anything +// else is dropped at collection time so the construction side never handles +// a name containing a path separator or dot-dot. +var safeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:+-]*$`) + +// SafeName reports whether name may be used as a sysfs entry name. +func SafeName(name string) bool { + return safeName.MatchString(name) +} + +// bdfRE matches a PCI function directory name ("0000:0c:00.0"). The domain +// is at least 4 hex digits: the kernel prints it with %04x, and synthetic +// domains above 0xffff (e.g. Intel VMD) print wider. +var bdfRE = regexp.MustCompile(`^[0-9a-f]{4,}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +// pciRootRE matches a PCI root complex directory name ("pci0000:07"). +var pciRootRE = regexp.MustCompile(`^pci[0-9a-f]{4,}:[0-9a-f]{2}$`) + +// IsBDF reports whether name is a PCI function directory name. +func IsBDF(name string) bool { return bdfRE.MatchString(name) } + +// Save serializes the snapshot to dst, creating parent directories. +func (s *Snapshot) Save(dst string) error { + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating %q: %w", dir, err) + } + b, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("marshaling RDMA sysfs snapshot: %w", err) + } + return os.WriteFile(dst, b, 0644) +} + +// Load deserializes a snapshot from src. Returns (nil, nil) if the file +// does not exist. +func Load(src string) (*Snapshot, error) { + b, err := os.ReadFile(src) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var s Snapshot + if err := json.Unmarshal(b, &s); err != nil { + return nil, fmt.Errorf("unmarshaling RDMA sysfs snapshot %q: %w", src, err) + } + return &s, nil +} diff --git a/pkg/rdma/snapshot_test.go b/pkg/rdma/snapshot_test.go new file mode 100644 index 00000000000..0eeb2fc3c8b --- /dev/null +++ b/pkg/rdma/snapshot_test.go @@ -0,0 +1,128 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdma + +import ( + "reflect" + "sort" + "testing" +) + +func TestSafeName(t *testing.T) { + for name, want := range map[string]bool{ + "uverbs0": true, + "mlx5_0": true, + "0000:0c:00.0": true, + "eth0.100": true, + "enp12s0f0np0": true, + "a.b": true, + "0": true, + "": false, + ".": false, + "..": false, + ".hidden": false, + "-flag": false, + "_x": false, + "a/b": false, + "a b": false, + "a\n": false, + "a\x00b": false, + } { + if got := SafeName(name); got != want { + t.Errorf("SafeName(%q) = %v, want %v", name, got, want) + } + } +} + +func TestIsBDF(t *testing.T) { + for name, want := range map[string]bool{ + "0000:0c:00.0": true, + "0000:ff:1f.7": true, + // Synthetic domains above 0xffff (e.g. Intel VMD) print wider than + // %04x. + "10000:e0:06.0": true, + "0000:0c:00": false, + "0000:0c:00.8": false, // function is 0-7 + "0000:0C:00.0": false, // the kernel prints lowercase hex + "000:0c:00.0": false, + "pci0000:0c": false, + "0000:0c:00.0/": false, + } { + if got := IsBDF(name); got != want { + t.Errorf("IsBDF(%q) = %v, want %v", name, got, want) + } + } +} + +func TestPCIRootRE(t *testing.T) { + for name, want := range map[string]bool{ + "pci0000:07": true, + "pci10000:e0": true, + "pci0000": false, + "0000:07": false, + "pci0000:07:00": false, + "pci0000:0G": false, + } { + if got := pciRootRE.MatchString(name); got != want { + t.Errorf("pciRootRE.MatchString(%q) = %v, want %v", name, got, want) + } + } +} + +func TestAddWithAncestors(t *testing.T) { + for _, tc := range []struct { + leaf string + want []string // nil means an error is expected + }{ + { + leaf: "devices/pci0000:07/0000:07:01.0/0000:0c:00.0", + want: []string{ + "devices/pci0000:07", + "devices/pci0000:07/0000:07:01.0", + "devices/pci0000:07/0000:07:01.0/0000:0c:00.0", + }, + }, + { + leaf: "devices/pci10000:e0/10000:e0:06.0", + want: []string{ + "devices/pci10000:e0", + "devices/pci10000:e0/10000:e0:06.0", + }, + }, + {leaf: "devices/pci0000:07/virtual0/0000:0c:00.0", want: nil}, + {leaf: "sys/devices/pci0000:07", want: nil}, + } { + set := make(map[string]bool) + err := addWithAncestors(set, tc.leaf) + if tc.want == nil { + if err == nil { + t.Errorf("addWithAncestors(%q) succeeded, want error", tc.leaf) + } + continue + } + if err != nil { + t.Errorf("addWithAncestors(%q) failed: %v", tc.leaf, err) + continue + } + var got []string + for p := range set { + got = append(got, p) + } + sort.Strings(got) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("addWithAncestors(%q) = %v, want %v", tc.leaf, got, tc.want) + } + } +} diff --git a/pkg/sentry/fsimpl/sys/BUILD b/pkg/sentry/fsimpl/sys/BUILD index e0399bfa730..6a4b589a9fb 100644 --- a/pkg/sentry/fsimpl/sys/BUILD +++ b/pkg/sentry/fsimpl/sys/BUILD @@ -22,6 +22,7 @@ go_library( "dir_refs.go", "kcov.go", "pci.go", + "rdma.go", "save_restore.go", "sys.go", ], @@ -35,6 +36,7 @@ go_library( "//pkg/fspath", "//pkg/fsutil", "//pkg/log", + "//pkg/rdma", "//pkg/refs", "//pkg/sentry/arch", "//pkg/sentry/fsimpl/host", @@ -61,6 +63,7 @@ go_test( deps = [ ":sys", "//pkg/abi/linux", + "//pkg/rdma", "//pkg/sentry/fsimpl/testutil", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", diff --git a/pkg/sentry/fsimpl/sys/rdma.go b/pkg/sentry/fsimpl/sys/rdma.go new file mode 100644 index 00000000000..27e12f3ba40 --- /dev/null +++ b/pkg/sentry/fsimpl/sys/rdma.go @@ -0,0 +1,361 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sys + +import ( + "fmt" + "path" + "strings" + + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/rdma" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" +) + +// This file builds the RDMA sysfs surface from a rdma.Snapshot. +// +// The layout is path-identical to the host: one canonical subtree under +// /sys/devices/pci... holds every attribute, with the device-class +// directories (infiniband, infiniband_verbs, net) as real directories under +// their PCI leaf, exactly as the kernel lays them out. /sys/class/* and +// /sys/bus/pci/devices contain only symlinks into the canonical subtree, +// with kernel-identical relative targets. Fidelity matters because +// consumers resolve these links with realpath() and then navigate the +// resolved paths: libibverbs discovers devices via +// /sys/class/infiniband_verbs, and NCCL's topology discovery realpath()s +// /sys/class/infiniband//device, walks the PCI hierarchy upward +// reading bridge attributes, and locates GPUs via +// "/sys/class/pci_bus//../../". +// +// Most attributes are static snapshots. Per-port dynamic state (link state, +// rate, and above all the RoCE GID table, which repopulates whenever the +// paired netdev's netns or addresses change) is served by reading host sysfs +// at access time; those host paths are bind-mounted into the sandbox's chroot. + +// rdmaSysfsDirs is the output of newRDMASysfs: subtrees for GetFilesystem +// to graft into the overall /sys hierarchy. +type rdmaSysfsDirs struct { + // devices maps the pciXXXX:YY root-complex names to their subtrees, + // to be added under /sys/devices. + devices map[string]kernfs.Inode + // class maps class-directory names (infiniband, infiniband_verbs, + // net, pci_bus) to symlink-farm directories for /sys/class. + class map[string]kernfs.Inode + // busPCIDevices contains the /sys/bus/pci/devices symlinks. + busPCIDevices map[string]kernfs.Inode + // node is the /sys/devices/system/node subtree, or nil. + node kernfs.Inode +} + +// rdmaDirTree is the intermediate mutable representation, keyed by entry name. +type rdmaDirTree struct { + children map[string]*rdmaDirTree + files map[string]string // static file contents + hostFiles map[string]string // file name -> host path read at access time + symlinks map[string]string // link name -> relative target +} + +func newRDMADirTree() *rdmaDirTree { + return &rdmaDirTree{ + children: map[string]*rdmaDirTree{}, + files: map[string]string{}, + hostFiles: map[string]string{}, + symlinks: map[string]string{}, + } +} + +// get returns the subtree at relPath (slash-separated), creating +// intermediate directories. +func (t *rdmaDirTree) get(relPath string) *rdmaDirTree { + cur := t + for _, part := range strings.Split(relPath, "/") { + if part == "" { + continue + } + next, ok := cur.children[part] + if !ok { + next = newRDMADirTree() + cur.children[part] = next + } + cur = next + } + return cur +} + +// newRDMASysfs builds the RDMA sysfs subtrees from snap. +func (fs *filesystem) newRDMASysfs(ctx context.Context, creds *auth.Credentials, snap *rdma.Snapshot) (*rdmaSysfsDirs, error) { + root := newRDMADirTree() + classIB := map[string]string{} // ibdev -> symlink target + classUverbs := map[string]string{} // uverbsN -> symlink target + classNet := map[string]string{} // netdev -> symlink target + classPCIBus := map[string]string{} // bus ("0000:0c") -> symlink target + + // 1. The canonical PCI hierarchy with per-level static attributes, plus + // the "subsystem" symlink every PCI device carries. NCCL and other + // consumers classify a directory as a PCI device by following + // subsystem to /sys/bus/pci; without it a device reads as "not a PCI + // device" and topology discovery aborts. The link points into our own + // synthesized /sys/bus/pci, so it is not a host escape hatch. The + // bus/ entry each device sits on is created for pci_bus below. + for _, n := range snap.PCINodes { + if err := checkRelPath(n.Path); err != nil { + return nil, err + } + d := root.get(n.Path) + for name, val := range n.Attrs { + if rdma.SafeName(name) { + d.files[name] = val + } + } + // Root complexes (pciXXXX:YY) carry no subsystem link and sit on + // no parent bus; only function directories (BDFs) do. + if rdma.IsBDF(path.Base(n.Path)) { + // depth of n.Path below /sys == number of "../" to reach /sys. + depth := strings.Count(n.Path, "/") + 1 + d.symlinks["subsystem"] = strings.Repeat("../", depth) + "bus/pci" + fs.addPCIBus(root, n.Path, classPCIBus) + } + } + + // 2. Per-device class subtrees under their PCI leaves. + for i := range snap.Devices { + dev := &snap.Devices[i] + if err := checkRelPath(dev.LeafPCI); err != nil { + return nil, err + } + if !rdma.SafeName(dev.IBDev) || !rdma.SafeName(dev.Uverbs) { + return nil, fmt.Errorf("unsafe RDMA device names %q/%q", dev.IBDev, dev.Uverbs) + } + // From //, the PCI function directory is + // three levels up plus its own name. + deviceLink := "../../../" + path.Base(dev.LeafPCI) + + // infiniband// + ibRel := dev.LeafPCI + "/infiniband/" + dev.IBDev + ib := root.get(ibRel) + for name, val := range dev.IBAttrs { + if rdma.SafeName(name) { + ib.files[name] = val + } + } + ib.symlinks["device"] = deviceLink + fs.addPorts(ib, dev) + classIB[dev.IBDev] = "../../" + ibRel + + // infiniband_verbs// + uvRel := dev.LeafPCI + "/infiniband_verbs/" + dev.Uverbs + uv := root.get(uvRel) + // IBDev is the trimmed device name (it is also a path component); the + // sysfs "ibdev" file holds that name followed by a newline. dev and + // abi_version were snapshotted verbatim, so they already carry the + // host's trailing newline. + uv.files["ibdev"] = dev.IBDev + "\n" + uv.files["abi_version"] = dev.ABIVersion + // The RDMA proxy registers the uverbs device at the host's fixed + // IB uverbs major (IB_UVERBS_MAJOR), so this host "major:minor" + // already matches the guest /dev/infiniband node and needs no rewrite. + uv.files["dev"] = dev.Dev + uv.symlinks["device"] = deviceLink + classUverbs[dev.Uverbs] = "../../" + uvRel + + // net// + for _, nd := range dev.NetDevs { + if !rdma.SafeName(nd.Name) { + continue + } + netRel := dev.LeafPCI + "/net/" + nd.Name + nt := root.get(netRel) + for name, val := range nd.Attrs { + if rdma.SafeName(name) { + nt.files[name] = val + } + } + nt.symlinks["device"] = deviceLink + classNet[nd.Name] = "../../" + netRel + } + } + // pci_bus and subsystem entries are created for every BDF node in the + // loop above; GPU leaves are already in snap.PCINodes, so they are + // covered without special handling here. + + // 3. Convert the tree and assemble the outputs. + out := &rdmaSysfsDirs{ + devices: map[string]kernfs.Inode{}, + class: map[string]kernfs.Inode{}, + busPCIDevices: map[string]kernfs.Inode{}, + } + devicesTree, ok := root.children["devices"] + if !ok { + return nil, fmt.Errorf("RDMA snapshot contains no devices/ paths") + } + for name, sub := range devicesTree.children { + out.devices[name] = fs.buildRDMADir(ctx, creds, sub) + } + + out.class["infiniband"] = fs.newDir(ctx, creds, defaultSysDirMode, fs.symlinkFarm(ctx, creds, classIB)) + uverbsEntries := fs.symlinkFarm(ctx, creds, classUverbs) + if snap.VerbsABIVersion != "" { + uverbsEntries["abi_version"] = fs.newStaticFile(ctx, creds, defaultSysMode, snap.VerbsABIVersion) + } + out.class["infiniband_verbs"] = fs.newDir(ctx, creds, defaultSysDirMode, uverbsEntries) + if len(classNet) > 0 { + out.class["net"] = fs.newDir(ctx, creds, defaultSysDirMode, fs.symlinkFarm(ctx, creds, classNet)) + } + if len(classPCIBus) > 0 { + out.class["pci_bus"] = fs.newDir(ctx, creds, defaultSysDirMode, fs.symlinkFarm(ctx, creds, classPCIBus)) + } + + for _, n := range snap.PCINodes { + base := path.Base(n.Path) + if rdma.IsBDF(base) { + out.busPCIDevices[base] = kernfs.NewStaticSymlink(ctx, creds, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), "../../../"+n.Path) + } + } + + if snap.NUMA != nil { + out.node = fs.buildNUMA(ctx, creds, snap.NUMA) + } + return out, nil +} + +// addPorts populates /ports// with static attributes and +// live host-backed files (GIDs, GID attributes, state, counters). +func (fs *filesystem) addPorts(ib *rdmaDirTree, dev *rdma.Device) { + for num, port := range dev.Ports { + if !rdma.SafeName(num) { + continue + } + hostBase := path.Join("/sys/class/infiniband", dev.IBDev, "ports", num) + p := ib.get("ports/" + num) + for name, val := range port.StaticAttrs { + if rdma.SafeName(name) { + p.files[name] = val + } + } + for _, name := range port.LiveAttrs { + if rdma.SafeName(name) { + p.hostFiles[name] = path.Join(hostBase, name) + } + } + gids := p.get("gids") + types := p.get("gid_attrs/types") + ndevs := p.get("gid_attrs/ndevs") + for _, idx := range port.GIDNames { + if !rdma.SafeName(idx) { + continue + } + gids.hostFiles[idx] = path.Join(hostBase, "gids", idx) + types.hostFiles[idx] = path.Join(hostBase, "gid_attrs/types", idx) + ndevs.hostFiles[idx] = path.Join(hostBase, "gid_attrs/ndevs", idx) + } + for _, name := range port.CounterNames { + if rdma.SafeName(name) { + p.get("counters").hostFiles[name] = path.Join(hostBase, "counters", name) + } + } + for _, name := range port.HWCounterNames { + if rdma.SafeName(name) { + p.get("hw_counters").hostFiles[name] = path.Join(hostBase, "hw_counters", name) + } + } + } +} + +// addPCIBus creates /pci_bus/ (the kernel places a +// bus directory inside the bridge that creates the bus) and records the +// /sys/class/pci_bus symlink. NCCL resolves GPU and NIC positions via +// "/sys/class/pci_bus//../../". +// +// Precondtion: rdma.IsBDF(path.Base(leaf)) == true +func (fs *filesystem) addPCIBus(root *rdmaDirTree, leaf string, classPCIBus map[string]string) { + base := path.Base(leaf) + i := strings.LastIndex(base, ":") + if i < 0 { + panic(fmt.Sprintf("RDMA leaf %q is not a PCI BDF", leaf)) + } + bus := base[:i] // "0000:0c" of "0000:0c:00.0" + parent := path.Dir(leaf) + root.get(parent + "/pci_bus/" + bus) + classPCIBus[bus] = "../../" + parent + "/pci_bus/" + bus +} + +func (fs *filesystem) buildNUMA(ctx context.Context, creds *auth.Credentials, numa *rdma.NUMA) kernfs.Inode { + entries := map[string]kernfs.Inode{} + for name, val := range numa.Aggregate { + if rdma.SafeName(name) { + entries[name] = fs.newStaticFile(ctx, creds, defaultSysMode, val) + } + } + for name, attrs := range numa.Nodes { + if !rdma.SafeName(name) { + continue + } + sub := map[string]kernfs.Inode{} + for attr, val := range attrs { + if rdma.SafeName(attr) { + sub[attr] = fs.newStaticFile(ctx, creds, defaultSysMode, val) + } + } + entries[name] = fs.newDir(ctx, creds, defaultSysDirMode, sub) + } + return fs.newDir(ctx, creds, defaultSysDirMode, entries) +} + +// buildRDMADir converts an rdmaDirTree into kernfs inodes. +func (fs *filesystem) buildRDMADir(ctx context.Context, creds *auth.Credentials, t *rdmaDirTree) kernfs.Inode { + entries := map[string]kernfs.Inode{} + for name, val := range t.files { + entries[name] = fs.newStaticFile(ctx, creds, defaultSysMode, val) + } + for name, hostPath := range t.hostFiles { + // Read from host sysfs at file-read time via openat(-1, ..., + // O_RDONLY|O_NOFOLLOW) (sys.hostFile.Generate); the rdmaproxy seccomp + // filter allows that openat. + entries[name] = fs.newHostFile(ctx, creds, defaultSysMode, hostPath) + } + for name, target := range t.symlinks { + entries[name] = kernfs.NewStaticSymlink(ctx, creds, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), target) + } + for name, child := range t.children { + entries[name] = fs.buildRDMADir(ctx, creds, child) + } + return fs.newDir(ctx, creds, defaultSysDirMode, entries) +} + +func (fs *filesystem) symlinkFarm(ctx context.Context, creds *auth.Credentials, targets map[string]string) map[string]kernfs.Inode { + entries := map[string]kernfs.Inode{} + for name, target := range targets { + if rdma.SafeName(name) { + entries[name] = kernfs.NewStaticSymlink(ctx, creds, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), target) + } + } + return entries +} + +// checkRelPath validates a snapshot-provided sysfs-relative path: rooted +// under devices/, with every component a safe name (see SafeName). +func checkRelPath(relPath string) error { + if !strings.HasPrefix(relPath, "devices/") { + return fmt.Errorf("RDMA sysfs path %q is not under devices/", relPath) + } + for _, part := range strings.Split(relPath, "/") { + if !rdma.SafeName(part) { + return fmt.Errorf("RDMA sysfs path %q contains unsafe component %q", relPath, part) + } + } + return nil +} diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index e81784b3f1a..03b358ef05e 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -29,6 +29,7 @@ import ( "gvisor.dev/gvisor/pkg/coverage" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/rdma" "gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" @@ -59,6 +60,11 @@ type InternalData struct { // EnableTPUProxyPaths is whether to populate sysfs paths used by hardware // accelerators. EnableTPUProxyPaths bool + // RDMASysfs, when non-nil, is the host sysfs snapshot from which the + // RDMA device topology (/sys/devices/pci..., /sys/class/infiniband*, + // /sys/class/net, /sys/class/pci_bus, /sys/bus/pci/devices, + // /sys/devices/system/node) is constructed. + RDMASysfs *rdma.Snapshot // TestSysfsPathPrefix is a prefix for the sysfs paths. It is useful for // unit testing. TestSysfsPathPrefix string @@ -120,18 +126,18 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt fsDirChildren["cgroup"] = fs.newCgroupDir(ctx, creds, defaultSysDirMode, nil) } - classSub := map[string]kernfs.Inode{ + classSub := map[string]kernfs.Inode{ // /sys/class "power_supply": fs.newDir(ctx, creds, defaultSysDirMode, nil), } - devicesSub := map[string]kernfs.Inode{ - "system": fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ - "cpu": cpuDir(ctx, fs, creds), - }), + devicesSub := map[string]kernfs.Inode{} // /sys/devices + systemSub := map[string]kernfs.Inode{ // /sys/devices/system + "cpu": cpuDir(ctx, fs, creds), } productName := "" - busSub := make(map[string]kernfs.Inode) - kernelSub := kernelDir(ctx, fs, creds) + busSub := make(map[string]kernfs.Inode) // /sys/bus + pciDevices := make(map[string]kernfs.Inode) // /sys/bus/pci/devices + kernelSub := kernelDir(ctx, fs, creds) // /sys/kernel if opts.InternalData != nil { idata := opts.InternalData.(*InternalData) productName = idata.ProductName @@ -166,9 +172,9 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt if err != nil { return nil, nil, err } - busSub["pci"] = fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ - "devices": fs.newDir(ctx, creds, defaultSysDirMode, pciDevicesSub), - }) + for name, sub := range pciDevicesSub { + pciDevices[name] = sub + } iommuGroups, err := fs.mirrorIOMMUGroups(ctx, creds, iommuGroupsPath, pciInfos) if err != nil { @@ -176,7 +182,39 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt } kernelSub["iommu_groups"] = fs.newDir(ctx, creds, defaultSysDirMode, iommuGroups) } + if idata.RDMASysfs != nil { + rdmaDirs, err := fs.newRDMASysfs(ctx, creds, idata.RDMASysfs) + if err != nil { + return nil, nil, err + } + for name, sub := range rdmaDirs.devices { + // The TPU-proxy devices and the RDMA ConnectX devices come from + // two different accelerator stacks and aren't exposed to the + // same sandbox today, so a shared /sys/devices root complex + // shouldn't occur. Deep-merging two sealed kernfs subtrees + // isn't supported, so reject an overlap. + if _, ok := devicesSub[name]; ok { + return nil, nil, fmt.Errorf("TPU proxy and RDMA sysfs both populate /sys/devices/%s", name) + } + devicesSub[name] = sub + } + for name, sub := range rdmaDirs.class { + classSub[name] = sub + } + for name, sub := range rdmaDirs.busPCIDevices { + pciDevices[name] = sub + } + if rdmaDirs.node != nil { + systemSub["node"] = rdmaDirs.node + } + } + } + if len(pciDevices) > 0 { + busSub["pci"] = fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ + "devices": fs.newDir(ctx, creds, defaultSysDirMode, pciDevices), + }) } + devicesSub["system"] = fs.newDir(ctx, creds, defaultSysDirMode, systemSub) if len(productName) > 0 { log.Debugf("Setting product_name: %q", productName) diff --git a/pkg/sentry/fsimpl/sys/sys_integration_test.go b/pkg/sentry/fsimpl/sys/sys_integration_test.go index 89785633242..ea3074554be 100644 --- a/pkg/sentry/fsimpl/sys/sys_integration_test.go +++ b/pkg/sentry/fsimpl/sys/sys_integration_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/go-cmp/cmp" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/rdma" "gvisor.dev/gvisor/pkg/sentry/fsimpl/sys" "gvisor.dev/gvisor/pkg/sentry/fsimpl/testutil" "gvisor.dev/gvisor/pkg/sentry/kernel" @@ -33,7 +34,7 @@ const ( vfioDev = "vfio-dev" ) -func newTestSystem(t *testing.T, pciTestDir string) *testutil.System { +func tryNewTestSystem(t *testing.T, idata *sys.InternalData) (*testutil.System, error) { k, err := testutil.Boot() if err != nil { t.Fatalf("Failed to create test kernel: %v", err) @@ -46,18 +47,30 @@ func newTestSystem(t *testing.T, pciTestDir string) *testutil.System { mountOpts := &vfs.MountOptions{ GetFilesystemOptions: vfs.GetFilesystemOptions{ - InternalData: &sys.InternalData{ - EnableTPUProxyPaths: pciTestDir != "", - TestSysfsPathPrefix: pciTestDir, - }, + InternalData: idata, }, } mns, err := k.VFS().NewMountNamespace(ctx, creds, "", sys.Name, mountOpts, nil) + if err != nil { + return nil, err + } + return testutil.NewSystem(ctx, t, k.VFS(), mns), nil +} + +func newTestSystemWithInternalData(t *testing.T, idata *sys.InternalData) *testutil.System { + s, err := tryNewTestSystem(t, idata) if err != nil { t.Fatalf("Failed to create new mount namespace: %v", err) } - return testutil.NewSystem(ctx, t, k.VFS(), mns) + return s +} + +func newTestSystem(t *testing.T, pciTestDir string) *testutil.System { + return newTestSystemWithInternalData(t, &sys.InternalData{ + EnableTPUProxyPaths: pciTestDir != "", + TestSysfsPathPrefix: pciTestDir, + }) } func TestReadCPUFile(t *testing.T) { @@ -309,3 +322,143 @@ func TestEnableTPUProxyPathsV5(t *testing.T) { }) } } + +// newRDMATestSnapshot returns a snapshot with a ConnectX NIC behind a bridge +// on domain 0000 and a GPU on an extended (VMD-style, 5-hex-digit) domain. +func newRDMATestSnapshot() *rdma.Snapshot { + const nicLeaf = "devices/pci0000:07/0000:07:01.0/0000:0c:00.0" + return &rdma.Snapshot{ + VerbsABIVersion: "6\n", + PCINodes: []rdma.PCINode{ + {Path: "devices/pci0000:07"}, + {Path: "devices/pci0000:07/0000:07:01.0", Attrs: map[string]string{"class": "0x060400\n"}}, + {Path: nicLeaf, Attrs: map[string]string{"vendor": "0x15b3\n", "uevent": "PCI_SLOT_NAME=0000:0c:00.0\n"}}, + {Path: "devices/pci10000:e0"}, + {Path: "devices/pci10000:e0/10000:e0:06.0", Attrs: map[string]string{"class": "0x030200\n", "vendor": "0x10de\n"}}, + }, + Devices: []rdma.Device{{ + Uverbs: "uverbs0", + IBDev: "mlx5_0", + LeafPCI: nicLeaf, + Dev: "231:192\n", + ABIVersion: "1\n", + IBAttrs: map[string]string{"node_guid": "0011:2233:4455:6677\n"}, + Ports: map[string]rdma.Port{ + "1": { + StaticAttrs: map[string]string{"link_layer": "Ethernet\n"}, + LiveAttrs: []string{"state"}, + GIDNames: []string{"0", "1"}, + CounterNames: []string{"port_rcv_data"}, + }, + }, + NetDevs: []rdma.NetDev{{Name: "eth1", Attrs: map[string]string{"mtu": "9000\n"}}}, + }}, + NUMA: &rdma.NUMA{ + Aggregate: map[string]string{"online": "0\n"}, + Nodes: map[string]map[string]string{"node0": {"cpulist": "0-3\n"}}, + }, + } +} + +func readTestFile(s *testutil.System, p string) (string, error) { + pop := s.PathOpAtRoot(p) + fd, err := s.VFS.OpenAt(s.Ctx, s.Creds, pop, &vfs.OpenOptions{}) + if err != nil { + return "", err + } + defer fd.DecRef(s.Ctx) + return s.ReadToEnd(fd) +} + +func TestRDMASysfs(t *testing.T) { + s := newTestSystemWithInternalData(t, &sys.InternalData{RDMASysfs: newRDMATestSnapshot()}) + defer s.Destroy() + + const nicLeaf = "/devices/pci0000:07/0000:07:01.0/0000:0c:00.0" + for p, want := range map[string]map[string]testutil.DirentType{ + "/devices/pci0000:07": { + "0000:07:01.0": linux.DT_DIR, + "pci_bus": linux.DT_DIR, + }, + "/devices/pci0000:07/0000:07:01.0": { + "0000:0c:00.0": linux.DT_DIR, + "pci_bus": linux.DT_DIR, + "subsystem": linux.DT_LNK, + "class": linux.DT_REG, + }, + nicLeaf: { + "infiniband": linux.DT_DIR, + "infiniband_verbs": linux.DT_DIR, + "net": linux.DT_DIR, + "subsystem": linux.DT_LNK, + "vendor": linux.DT_REG, + "uevent": linux.DT_REG, + }, + nicLeaf + "/infiniband/mlx5_0": { + "device": linux.DT_LNK, + "node_guid": linux.DT_REG, + "ports": linux.DT_DIR, + }, + nicLeaf + "/infiniband/mlx5_0/ports/1": { + "link_layer": linux.DT_REG, + "state": linux.DT_REG, + "gids": linux.DT_DIR, + "gid_attrs": linux.DT_DIR, + "counters": linux.DT_DIR, + }, + nicLeaf + "/infiniband/mlx5_0/ports/1/gids": { + "0": linux.DT_REG, + "1": linux.DT_REG, + }, + nicLeaf + "/infiniband_verbs/uverbs0": { + "ibdev": linux.DT_REG, + "abi_version": linux.DT_REG, + "dev": linux.DT_REG, + "device": linux.DT_LNK, + }, + nicLeaf + "/net/eth1": { + "mtu": linux.DT_REG, + "device": linux.DT_LNK, + }, + // Extended-domain (VMD-style) GPU root. + "/devices/pci10000:e0": { + "10000:e0:06.0": linux.DT_DIR, + "pci_bus": linux.DT_DIR, + }, + "/class/infiniband": {"mlx5_0": linux.DT_LNK}, + "/class/infiniband_verbs": {"uverbs0": linux.DT_LNK, "abi_version": linux.DT_REG}, + "/class/net": {"eth1": linux.DT_LNK}, + "/class/pci_bus": { + "0000:07": linux.DT_LNK, + "0000:0c": linux.DT_LNK, + "10000:e0": linux.DT_LNK, + }, + "/bus/pci/devices": { + "0000:07:01.0": linux.DT_LNK, + "0000:0c:00.0": linux.DT_LNK, + "10000:e0:06.0": linux.DT_LNK, + }, + "/devices/system/node": {"online": linux.DT_REG, "node0": linux.DT_DIR}, + "/devices/system/node/node0": {"cpulist": linux.DT_REG}, + } { + pop := s.PathOpAtRoot(p) + s.AssertAllDirentTypes(s.ListDirents(pop), want) + } + + // Attribute contents, including via a /sys/class symlink (the path + // consumers actually resolve). + for p, want := range map[string]string{ + "/class/infiniband/mlx5_0/node_guid": "0011:2233:4455:6677\n", + "/class/infiniband_verbs/abi_version": "6\n", + nicLeaf + "/infiniband_verbs/uverbs0/ibdev": "mlx5_0\n", + "/class/infiniband/mlx5_0/ports/1/../../device/vendor": "0x15b3\n", + } { + got, err := readTestFile(s, p) + if err != nil { + t.Fatalf("reading %q: %v", p, err) + } + if got != want { + t.Errorf("%q contains %q, want %q", p, got, want) + } + } +} diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 28535dd3cb2..b67342bfb8f 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -56,6 +56,7 @@ go_library( "//pkg/memutil", "//pkg/metric", "//pkg/rand", + "//pkg/rdma", "//pkg/refs", "//pkg/sentry/arch", "//pkg/sentry/arch:registers_go_proto", diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 5a6018cbb90..c2f2575e078 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -38,6 +38,7 @@ import ( "gvisor.dev/gvisor/pkg/memutil" "gvisor.dev/gvisor/pkg/metric" "gvisor.dev/gvisor/pkg/rand" + "gvisor.dev/gvisor/pkg/rdma" "gvisor.dev/gvisor/pkg/refs" "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" @@ -247,6 +248,10 @@ type Loader struct { // /sys/devices/virtual/dmi/id/product_name. productName string + // rdmaSysfs is the host sysfs snapshot for RDMA device topology, or + // nil when disabled. + rdmaSysfs *rdma.Snapshot + // cpuQuota and cpuPeriod are the raw host CFS settings that should be // exposed through sandbox cgroupfs. cpuQuota int64 @@ -422,6 +427,10 @@ type Args struct { // ProductName is the value to show in // /sys/devices/virtual/dmi/id/product_name. ProductName string + + // RDMASysfs is the host sysfs snapshot for RDMA device topology, or + // nil when disabled. + RDMASysfs *rdma.Snapshot // PodInitConfigFD is the file descriptor to a file passed in the // --pod-init-config flag PodInitConfigFD int @@ -538,6 +547,7 @@ func New(args Args) (*Loader, error) { sharedMounts: make(map[string]*vfs.Mount), stopProfiling: stopProfiling, productName: args.ProductName, + rdmaSysfs: args.RDMASysfs, cpuQuota: args.CPUQuota, cpuPeriod: args.CPUPeriod, hostTHP: args.HostTHP, diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 1ff3f9bfd8f..7a636943d4a 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -36,6 +36,7 @@ import ( "gvisor.dev/gvisor/pkg/fspath" "gvisor.dev/gvisor/pkg/fsutil" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/rdma" "gvisor.dev/gvisor/pkg/sentry/checkpoint" "gvisor.dev/gvisor/pkg/sentry/devices/memdev" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" @@ -921,7 +922,7 @@ func (c *containerMounter) getPathMode(ctx context.Context, creds *auth.Credenti } func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *mountInfo) (*vfs.Mount, error) { - fsName, opts, err := getMountNameAndOptions(spec, conf, submount, c.l.productName, c.containerName, c.containerID, c.l.fsRestore) + fsName, opts, err := getMountNameAndOptions(spec, conf, submount, c.l.productName, c.containerName, c.containerID, c.l.fsRestore, c.l.rdmaSysfs) if err != nil { return nil, fmt.Errorf("mountOptions failed: %w", err) } @@ -984,7 +985,7 @@ func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec, // getMountNameAndOptions retrieves the fsName, opts, and useOverlay values // used for mounts. -func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, productName, containerName, containerID string, fsr *fsRestore) (string, *vfs.MountOptions, error) { +func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, productName, containerName, containerID string, fsr *fsRestore, rdmaSysfs *rdma.Snapshot) (string, *vfs.MountOptions, error) { fsName := m.mount.Type var ( mopts = m.mount.Options @@ -1004,7 +1005,10 @@ func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, internalData = newProcInternalData(conf, spec) case sys.Name: - sysData := &sys.InternalData{EnableTPUProxyPaths: specutils.TPUProxyEnabled(spec, conf)} + sysData := &sys.InternalData{ + EnableTPUProxyPaths: specutils.TPUProxyEnabled(spec, conf), + RDMASysfs: rdmaSysfs, + } if len(productName) > 0 { sysData.ProductName = productName } @@ -1385,7 +1389,7 @@ func (c *containerMounter) mountSharedMaster(ctx context.Context, spec *specs.Sp // Mount the master using the options from the hint (mount annotations). origOpts := mntInfo.mount.Options mntInfo.mount.Options = mntInfo.hint.Mount.Options - fsName, opts, err := getMountNameAndOptions(spec, conf, mntInfo, c.l.productName, c.containerName, c.containerID, c.l.fsRestore) + fsName, opts, err := getMountNameAndOptions(spec, conf, mntInfo, c.l.productName, c.containerName, c.containerID, c.l.fsRestore, c.l.rdmaSysfs) mntInfo.mount.Options = origOpts if err != nil { return nil, err diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD index 31269e0d4be..e53691914a3 100644 --- a/runsc/cmd/BUILD +++ b/runsc/cmd/BUILD @@ -72,6 +72,7 @@ go_library( "//pkg/log", "//pkg/metric", "//pkg/prometheus", + "//pkg/rdma", "//pkg/ring0", "//pkg/sentry/checkpoint", "//pkg/sentry/control", diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index c50a2d5f3f8..edd1d71ac93 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -36,6 +36,7 @@ import ( "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/metric" "gvisor.dev/gvisor/pkg/prometheus" + "gvisor.dev/gvisor/pkg/rdma" "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/hostmm" @@ -581,6 +582,15 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma linux.SetAFSSyscallPanic(conf.TestOnlyAFSSyscallPanic) + var rdmaSnap *rdma.Snapshot + if specutils.RDMAEnabled(spec, conf) { + // Load the RDMA sysfs snapshot serialized by the chroot stage. + var err error + if rdmaSnap, err = rdma.Load(rdma.Path); err != nil { + util.Fatalf("loading RDMA sysfs snapshot: %v", err) + } + } + // Create the loader. bootArgs := boot.Args{ ID: f.Arg(0), @@ -601,6 +611,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma TotalMem: b.totalMem, TotalHostMem: b.totalHostMem, UserLogFD: b.userLogFD, + RDMASysfs: rdmaSnap, ProductName: b.productName, PodInitConfigFD: b.podInitConfigFD, SinkFDs: b.sinkFDs.GetArray(), diff --git a/runsc/cmd/chroot.go b/runsc/cmd/chroot.go index 6362dbf0520..f6f41c0e0d3 100644 --- a/runsc/cmd/chroot.go +++ b/runsc/cmd/chroot.go @@ -25,6 +25,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/tpu" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/rdma" "gvisor.dev/gvisor/runsc/cmd/sandboxsetup" "gvisor.dev/gvisor/runsc/cmd/util" "gvisor.dev/gvisor/runsc/config" @@ -43,6 +44,21 @@ func mountInChroot(chroot, src, dst, typ string, flags uint32) error { return nil } +// bindMountSysfsInChrootReadonly bind-mounts the host sysfs directory src at +// dst inside the chroot read-only. mount(2) ignores attribute flags like +// MS_RDONLY when MS_BIND is set, so the bind must be followed by a remount +// that applies them. The remount also carries sysfs's standard attribute +// flags (nosuid,nodev,noexec), which the bind inherited and a remount may not +// clear. +func bindMountSysfsInChrootReadonly(chroot, src, dst string) error { + if err := mountInChroot(chroot, src, dst, "bind", unix.MS_BIND); err != nil { + return err + } + return specutils.SafeMount("", filepath.Join(chroot, dst), "", + unix.MS_REMOUNT|unix.MS_BIND|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC, + "", "/proc") +} + // setupMinimalProcfs creates a minimal procfs-like tree at `${chroot}/proc`. func setupMinimalProcfs(chroot string) error { // We can't always directly mount procfs because it may be obstructed @@ -148,6 +164,10 @@ func setUpChroot(spec *specs.Spec, conf *config.Config) error { return fmt.Errorf("error configuring chroot for TPU devices: %w", err) } + if err := rdmaSysfsUpdateChroot(chroot, spec, conf); err != nil { + return fmt.Errorf("error configuring chroot for RDMA sysfs: %w", err) + } + if err := specutils.SafeMount("", chroot, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_BIND, "", "/proc"); err != nil { return fmt.Errorf("error remounting chroot in read-only: %v", err) } @@ -171,7 +191,7 @@ func tpuProxyUpdateChroot(hostRoot, chroot string, spec *specs.Spec, conf *confi } devNum := path.Base(devPath) iommuGroupPath := path.Join("/sys/kernel/iommu_groups", devNum) - if err := mountInChroot(chroot, path.Join(hostRoot, iommuGroupPath), iommuGroupPath, "bind", unix.MS_BIND|unix.MS_RDONLY); err != nil { + if err := bindMountSysfsInChrootReadonly(chroot, path.Join(hostRoot, iommuGroupPath), iommuGroupPath); err != nil { return fmt.Errorf("error mounting %q in chroot: %v", iommuGroupPath, err) } allowedDeviceIDs[tpu.TPUV5pDeviceID] = struct{}{} @@ -202,7 +222,7 @@ func tpuProxyUpdateChroot(hostRoot, chroot string, spec *specs.Spec, conf *confi if err := filepath.WalkDir(sysDevicesPath, func(path string, d os.DirEntry, err error) error { if d.Type().IsDir() && util.IsPCIDeviceDirTPU(path, allowedDeviceIDs) { chrootPath := strings.Replace(path, hostRoot, "/", 1) - if err := mountInChroot(chroot, path, chrootPath, "bind", unix.MS_BIND|unix.MS_RDONLY); err != nil { + if err := bindMountSysfsInChrootReadonly(chroot, path, chrootPath); err != nil { return fmt.Errorf("error mounting %q in chroot: %v", path, err) } } @@ -213,3 +233,73 @@ func tpuProxyUpdateChroot(hostRoot, chroot string, spec *specs.Spec, conf *confi } return err } + +// rdmaSysfsUpdateChroot collects the host's RDMA sysfs state into a JSON +// snapshot saved in the sandbox chroot, from which the sentry later builds the +// sandbox's virtual RDMA sysfs tree. Per-port dynamic files (GID tables, port +// state, counters) are not snapshotted; instead the host's per-ibdev "ports" +// subtrees are bind-mounted into the chroot read-only so the sentry can serve +// them live. +// +// Preconditions: +// - Must run before pivot_root, while the host /sys is still reachable. +// - The RDMA netdevs must still be in the host netns, so that they and their +// /sys/class/net/ attributes are visible in the host sysfs read here. +// +// The RDMA netdevs must therefore stay in the host netns until this collection +// runs; moving them into the sandbox netns should only be done afterwards. +// +// We deliberately do not support the reverse ordering - a sandbox netns +// pre-created and populated with the RDMA netdevs before `runsc create` - for +// two reasons: +// +// - /sys/class/net entries are netns-tagged: a sysfs mount shows only the +// netdevs of the netns that mounted it. Once a netdev is in the sandbox +// netns it is gone from the host sysfs we read here, so its attributes +// could only be collected via a sysfs mount tagged with the sandbox netns. +// - The sandbox cannot make such a mount. Mounting a fresh sysfs requires the +// current netns to be owned by the current userns (the kernel's +// net_ns_current_may_mount check: net_ns->user_ns == current_user_ns). +// Under directfs the sandbox runs in a new userns that does not own the +// pre-created netns (that netns is owned by the host userns), so the mount +// fails with EPERM. +// +// The per-ibdev "ports" bind mounts, by contrast, keep working after the +// netdevs move, because the InfiniBand class is not netns-tagged when the RDMA +// subsystem is in "rdma system netns shared" mode (a requirement for RDMA +// support here). GID tables and port state therefore stay readable through +// those mounts. +func rdmaSysfsUpdateChroot(chroot string, spec *specs.Spec, conf *config.Config) error { + if !specutils.RDMAEnabled(spec, conf) { + return nil + } + snap, err := rdma.Collect("/sys", specutils.UverbsDevicesInSpec(spec)) + if err != nil { + return fmt.Errorf("collecting RDMA sysfs snapshot: %w", err) + } + if snap == nil { + return nil + } + if err := snap.Save(filepath.Join(chroot, rdma.Path)); err != nil { + return fmt.Errorf("saving RDMA sysfs snapshot: %w", err) + } + for i := range snap.Devices { + // Bind-mount the host's per-ibdev "ports" subtree inside the chroot at + // /sys/class/infiniband//ports. On the host + // /sys/class/infiniband/ is a symlink into /sys/devices, and + // the bind mount resolves it, so the real ports directory is what + // lands in the chroot. In the sandbox the sentry builds the canonical + // tree at /sys/devices/pci.../infiniband// (with + // /sys/class/infiniband/ as a symlink to it, mirroring the + // host), and each live port file there records this + // /sys/class/infiniband//ports/... path as its host source; the + // sentry opens that path to serve the file's contents live. So the + // class path here need NOT match the sandbox's canonical (devices) + // path since it only has to be where the sentry looks. + portsPath := filepath.Join("/sys/class/infiniband", snap.Devices[i].IBDev, "ports") + if err := bindMountSysfsInChrootReadonly(chroot, portsPath, portsPath); err != nil { + return fmt.Errorf("bind-mounting %q read-only in chroot: %w", portsPath, err) + } + } + return nil +} diff --git a/runsc/config/config.go b/runsc/config/config.go index 6da45f53b79..334ac7fb508 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -376,6 +376,12 @@ type Config struct { // TPUProxy enables support for TPUs. TPUProxy bool `flag:"tpuproxy"` + // RDMAProxy enables RDMA support for containers with + // /dev/infiniband/uverbs* devices. It currently gates only the RDMA + // device sysfs topology; the verbs device proxy that makes it usable is + // still in development. + RDMAProxy bool `flag:"rdmaproxy"` + // TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in // tests. It allows runsc to start the sandbox process as the current // user, and without chrooting the sandbox process. This can be diff --git a/runsc/config/flags.go b/runsc/config/flags.go index e0d5c4e101f..f0e197bb9ba 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -183,6 +183,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.String("nvproxy-driver-version", "", "NVIDIA driver ABI version to use. If empty, autodetect installed driver version. The special value 'latest' may also be used to use the latest ABI.") flagSet.Bool("nvproxy-allow-unsupported-driver", false, "allow nvproxy to be initialized with an unsupported driver version.") flagSet.String("nvproxy-allowed-driver-capabilities", "utility,compute", "Comma separated list of NVIDIA driver capabilities that are allowed to be requested by the container. If 'all' is specified here, it is resolved to all driver capabilities supported in nvproxy. If 'all' is requested by the container, it is resolved to this list.") + flagSet.Bool("rdmaproxy", false, "WIP: enable RDMA support for containers with /dev/infiniband/uverbs* devices.") flagSet.Bool("tpuproxy", false, "LEGACY: enable support for TPU devices. TPU support gets automatically enabled if TPU devices are present in the OCI spec.") // Test flags, not to be used outside tests, ever. diff --git a/runsc/specutils/BUILD b/runsc/specutils/BUILD index 420931de017..bac4e4ecfec 100644 --- a/runsc/specutils/BUILD +++ b/runsc/specutils/BUILD @@ -14,6 +14,7 @@ go_library( "hooks.go", "namespace.go", "nvidia.go", + "rdma.go", "restore.go", "specutils.go", ], @@ -22,6 +23,7 @@ go_library( "//pkg/abi/linux", "//pkg/abi/nvgpu", "//pkg/log", + "//pkg/rdma", "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/kernel/auth", "//runsc/config", diff --git a/runsc/specutils/rdma.go b/runsc/specutils/rdma.go new file mode 100644 index 00000000000..a09ff2037f3 --- /dev/null +++ b/runsc/specutils/rdma.go @@ -0,0 +1,63 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package specutils + +import ( + "path" + "strings" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/rdma" + "gvisor.dev/gvisor/runsc/config" +) + +// UverbsDevicesInSpec returns the /dev/infiniband/uverbs* devices listed in +// the OCI spec, in spec order. +func UverbsDevicesInSpec(spec *specs.Spec) []rdma.UverbsSpec { + if spec.Linux == nil { + return nil + } + var out []rdma.UverbsSpec + for _, dev := range spec.Linux.Devices { + if !strings.HasPrefix(dev.Path, "/dev/infiniband/uverbs") { + continue + } + out = append(out, rdma.UverbsSpec{ + Name: path.Base(dev.Path), + Major: dev.Major, + Minor: dev.Minor, + }) + } + return out +} + +// RDMAEnabled returns true if the sandbox should set up RDMA support. +func RDMAEnabled(spec *specs.Spec, conf *config.Config) bool { + return conf.RDMAProxy && specHasUverbsDevice(spec) +} + +// specHasUverbsDevice reports whether the spec lists any +// /dev/infiniband/uverbs* device. +func specHasUverbsDevice(spec *specs.Spec) bool { + if spec.Linux == nil { + return false + } + for _, dev := range spec.Linux.Devices { + if strings.HasPrefix(dev.Path, "/dev/infiniband/uverbs") { + return true + } + } + return false +}