diff --git a/g3doc/user_guide/BUILD b/g3doc/user_guide/BUILD index 68b6017c01a..4abcf3e3163 100644 --- a/g3doc/user_guide/BUILD +++ b/g3doc/user_guide/BUILD @@ -81,6 +81,14 @@ doc( weight = "51", ) +doc( + name = "rdma", + src = "rdma.md", + category = "User Guide", + permalink = "/docs/user_guide/rdma/", + weight = "52", +) + doc( name = "runtime_monitoring", src = "runtime_monitoring.md", diff --git a/g3doc/user_guide/rdma.md b/g3doc/user_guide/rdma.md new file mode 100644 index 00000000000..a946d4cb1af --- /dev/null +++ b/g3doc/user_guide/rdma.md @@ -0,0 +1,70 @@ +# RDMA Support + +[TOC] + +gVisor supports RDMA (Remote Direct Memory Access) networking, allowing +sandboxed applications to use high-performance InfiniBand/RoCE hardware — for +example, multi-node NCCL collectives in distributed AI/ML training — while +keeping the host isolated from the workload. + +To achieve this, gVisor implements a proxy driver inside the sandbox, +henceforth referred to as `rdmaproxy`. `rdmaproxy` proxies the application's +interactions with the host's RDMA driver, giving the sandboxed application +access to the RDMA verbs devices (`/dev/infiniband/uverbs*`) declared in its +OCI spec. The application — and libraries such as `libibverbs` and NCCL — can +run unmodified inside the sandbox and interact transparently with these +devices. gVisor also constructs a faithful view of the host's RDMA topology +under `/sys` so that device-discovery and topology-detection logic behaves the +same inside the sandbox as on the host. + +RDMA support is enabled with the `--rdmaproxy` flag and applies to containers +whose OCI spec lists one or more `/dev/infiniband/uverbs*` devices. + +When combined with GPUs (see [GPU Support](gpu.md)), gVisor supports +GPUDirect RDMA, allowing the NIC to transfer data directly to and from GPU +memory. + +## Limitations + +RDMA support is under active development. The following limitations apply: + +* **Mellanox and AWS EFA NICs only.** Only Mellanox ConnectX (`mlx5`) + adapters and AWS Elastic Fabric Adapters (`efa`) are currently supported. + Support for additional vendors is planned. + +* **Host kernel 5.12 or newer.** `rdmaproxy` proxies the modern + `RDMA_VERBS_IOCTL` interface only; the legacy `write(2)` command + interface is not supported. The newest uverbs ioctl method it relies on, + dma-buf memory registration for GPUDirect RDMA, landed in Linux 5.12. + +* **GPUDirect uses dma-buf only.** GPU memory is registered with the NIC + through the dma-buf mechanism, which is the modern default. The legacy + `nvidia-peermem` kernel-module path is not supported. + +* **EFA needs a dma-buf-capable NCCL plugin.** Because GPUDirect works only + through dma-buf (above), AWS EFA requires `aws-ofi-nccl` v1.19.2 or newer: + earlier releases hard-disable dma-buf on EFA device generations 1-3 and + register GPU memory by virtual address instead, which is not supported. + +* **Single-container sandboxes only.** The RDMA devices must be declared in + the OCI spec of the sandbox's root container. Deployments where the + devices appear only in a sub-container's spec — such as a Kubernetes pod + where the RDMA devices belong to an application container rather than the + pod's root — are not yet supported. + +* **Device lifecycle at sandbox creation.** The RDMA network devices must + still reside in the host network namespace when the sandbox is created + (`runsc create`), and must not be pre-placed into the sandbox's network + namespace beforehand. They should then be moved, fully configured, into + the sandbox's network namespace before the application connects RDMA + queue pairs — for example via an OCI `createRuntime` hook or between + `runsc create` and `runsc start`. Such setup is not unusual; Docker seems + to be doing this as well. + +* **RoCE requires shared RDMA namespace mode.** For RoCE (RDMA over + Converged Ethernet) devices, the host's RDMA subsystem must be in the + shared network-namespace mode (`rdma system set netns shared`), which is + the default on most systems. + +* **No checkpoint/restore.** Sandboxes using RDMA devices cannot be + checkpointed. diff --git a/pkg/abi/ib/BUILD b/pkg/abi/ib/BUILD new file mode 100644 index 00000000000..393d9834f88 --- /dev/null +++ b/pkg/abi/ib/BUILD @@ -0,0 +1,20 @@ +load("//tools:defs.bzl", "go_library") + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +go_library( + name = "ib", + srcs = ["ib.go"], + hostlayout = True, + marshal = True, + visibility = [ + "//pkg/sentry:internal", + "//runsc:__subpackages__", + ], + deps = [ + "//pkg/abi/linux", + "//pkg/marshal", + ], +) diff --git a/pkg/abi/ib/ib.go b/pkg/abi/ib/ib.go new file mode 100644 index 00000000000..09f7b6710e2 --- /dev/null +++ b/pkg/abi/ib/ib.go @@ -0,0 +1,344 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ib contains constants and structs from the Linux InfiniBand +// userspace verbs ABI (include/uapi/rdma/). +package ib + +import ( + "structs" + + "gvisor.dev/gvisor/pkg/abi/linux" +) + +// IB_UVERBS_MAJOR is the fixed char-device major the Linux InfiniBand uverbs +// subsystem uses for /dev/infiniband/uverbs* (defined in +// drivers/infiniband/core/uverbs_main.c). +// +// The first 32 uverbs devices (minors 192-223) use this fixed major; a host +// with more than 32 RDMA devices would assign later ones a dynamic major. +const IB_UVERBS_MAJOR = 231 + +// RDMAVerbsIoctl is RDMA_VERBS_IOCTL, the single ioctl command of the modern +// uverbs interface (include/uapi/rdma/rdma_user_ioctl.h). +var RDMAVerbsIoctl = linux.IOWR(0x1b, 1, SizeofUverbsIoctlHdr) + +// Struct sizes. +var ( + SizeofUverbsIoctlHdr = uint32((*UverbsIoctlHdr)(nil).SizeBytes()) + SizeofUverbsAttr = uint32((*UverbsAttr)(nil).SizeBytes()) +) + +// UverbsIoctlHdr is struct ib_uverbs_ioctl_hdr, the fixed header of a +// RDMA_VERBS_IOCTL request; NumAttrs UverbsAttr follow it +// (include/uapi/rdma/rdma_user_ioctl_cmds.h). +// +// +marshal +type UverbsIoctlHdr struct { + _ structs.HostLayout + Length uint16 + ObjectID uint16 + MethodID uint16 + NumAttrs uint16 + Reserved1 uint64 + DriverID uint32 + Reserved2 uint32 +} + +// UverbsAttr is struct ib_uverbs_attr, one attribute of a RDMA_VERBS_IOCTL +// request (include/uapi/rdma/rdma_user_ioctl_cmds.h). Data carries an inline +// value, a user pointer (Len > 8), an object handle, or an fd, depending on +// the attribute. +// +// +marshal slice:UverbsAttrSlice +type UverbsAttr struct { + _ structs.HostLayout + AttrID uint16 + Len uint16 + Flags uint16 + AttrData uint16 + Data uint64 +} + +// UVERBS ioctl object, method, and attribute IDs from +// include/uapi/rdma/ib_user_ioctl_cmds.h. Only the entries the rdma proxy +// models are listed; the enums are cherry-picked, so values are explicit. + +// enum uverbs_default_objects. +const ( + UVERBS_OBJECT_DEVICE = 0 + UVERBS_OBJECT_PD = 1 + UVERBS_OBJECT_CQ = 3 + UVERBS_OBJECT_QP = 4 + UVERBS_OBJECT_AH = 6 + UVERBS_OBJECT_MR = 7 + UVERBS_OBJECT_ASYNC_EVENT = 16 +) + +// Method IDs within each object namespace. +const ( + // enum uverbs_methods_device. + UVERBS_METHOD_INVOKE_WRITE = 0 + UVERBS_METHOD_QUERY_PORT = 2 + UVERBS_METHOD_GET_CONTEXT = 3 + UVERBS_METHOD_QUERY_CONTEXT = 4 + UVERBS_METHOD_QUERY_GID_TABLE = 5 + UVERBS_METHOD_QUERY_GID_ENTRY = 6 + + // enum uverbs_methods_pd. + UVERBS_METHOD_PD_DESTROY = 0 + + // enum uverbs_methods_cq. + UVERBS_METHOD_CQ_CREATE = 0 + UVERBS_METHOD_CQ_DESTROY = 1 + + // enum uverbs_methods_qp. + UVERBS_METHOD_QP_CREATE = 0 + UVERBS_METHOD_QP_DESTROY = 1 + + // enum uverbs_methods_ah. AH creation uses the legacy write path + // (IB_USER_VERBS_CMD_CREATE_AH via INVOKE_WRITE); only destroy has a + // modern object method. + UVERBS_METHOD_AH_DESTROY = 0 + + // enum uverbs_methods_mr. + UVERBS_METHOD_MR_DESTROY = 1 + UVERBS_METHOD_REG_DMABUF_MR = 4 + UVERBS_METHOD_REG_MR = 5 + + // enum uverbs_method_async_event. + UVERBS_METHOD_ASYNC_EVENT_ALLOC = 0 +) + +// enum uverbs_attrs_invoke_write_cmd_attr_ids. +const ( + UVERBS_ATTR_CORE_IN = 0 + UVERBS_ATTR_CORE_OUT = 1 + UVERBS_ATTR_WRITE_CMD = 2 +) + +// enum uverbs_attrs_get_context_attr_ids. +const ( + UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS = 0 + UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT = 1 + UVERBS_ATTR_GET_CONTEXT_FD_ARR = 2 +) + +// enum uverbs_attrs_query_context_attr_ids. +const ( + UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS = 0 + UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT = 1 +) + +// enum uverbs_attrs_query_port_cmd_attr_ids. +const ( + UVERBS_ATTR_QUERY_PORT_PORT_NUM = 0 + UVERBS_ATTR_QUERY_PORT_RESP = 1 +) + +// enum uverbs_attrs_query_gid_table_cmd_attr_ids. +const ( + UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE = 0 + UVERBS_ATTR_QUERY_GID_TABLE_FLAGS = 1 + UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES = 2 + UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES = 3 +) + +// enum uverbs_attrs_query_gid_entry_cmd_attr_ids. +const ( + UVERBS_ATTR_QUERY_GID_ENTRY_PORT = 0 + UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX = 1 + UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS = 2 + UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY = 3 +) + +// enum uverbs_attrs_destroy_pd_cmd_attr_ids. +const UVERBS_ATTR_DESTROY_PD_HANDLE = 0 + +// enum uverbs_attrs_ah_destroy_ids. +const UVERBS_ATTR_DESTROY_AH_HANDLE = 0 + +// enum uverbs_attrs_reg_mr_cmd_attr_ids. +const ( + UVERBS_ATTR_REG_MR_HANDLE = 0 + UVERBS_ATTR_REG_MR_PD_HANDLE = 1 + UVERBS_ATTR_REG_MR_DMA_HANDLE = 2 + UVERBS_ATTR_REG_MR_IOVA = 3 + UVERBS_ATTR_REG_MR_ADDR = 4 + UVERBS_ATTR_REG_MR_LENGTH = 5 + UVERBS_ATTR_REG_MR_ACCESS_FLAGS = 6 + UVERBS_ATTR_REG_MR_FD = 7 + UVERBS_ATTR_REG_MR_FD_OFFSET = 8 + UVERBS_ATTR_REG_MR_RESP_LKEY = 9 + UVERBS_ATTR_REG_MR_RESP_RKEY = 10 +) + +// enum uverbs_attrs_reg_dmabuf_mr_cmd_attr_ids. +const ( + UVERBS_ATTR_REG_DMABUF_MR_HANDLE = 0 + UVERBS_ATTR_REG_DMABUF_MR_PD_HANDLE = 1 + UVERBS_ATTR_REG_DMABUF_MR_OFFSET = 2 + UVERBS_ATTR_REG_DMABUF_MR_LENGTH = 3 + UVERBS_ATTR_REG_DMABUF_MR_IOVA = 4 + UVERBS_ATTR_REG_DMABUF_MR_FD = 5 + UVERBS_ATTR_REG_DMABUF_MR_ACCESS_FLAGS = 6 + UVERBS_ATTR_REG_DMABUF_MR_RESP_LKEY = 7 + UVERBS_ATTR_REG_DMABUF_MR_RESP_RKEY = 8 +) + +// enum uverbs_attrs_destroy_mr_cmd_attr_ids. +const UVERBS_ATTR_DESTROY_MR_HANDLE = 0 + +// enum uverbs_attrs_create_cq_cmd_attr_ids. +const ( + UVERBS_ATTR_CREATE_CQ_HANDLE = 0 + UVERBS_ATTR_CREATE_CQ_CQE = 1 + UVERBS_ATTR_CREATE_CQ_USER_HANDLE = 2 + UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL = 3 + UVERBS_ATTR_CREATE_CQ_COMP_VECTOR = 4 + UVERBS_ATTR_CREATE_CQ_FLAGS = 5 + UVERBS_ATTR_CREATE_CQ_RESP_CQE = 6 + UVERBS_ATTR_CREATE_CQ_EVENT_FD = 7 + UVERBS_ATTR_CREATE_CQ_BUFFER_VA = 8 + UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH = 9 + UVERBS_ATTR_CREATE_CQ_BUFFER_FD = 10 + UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET = 11 + UVERBS_ATTR_CREATE_CQ_BUF_UMEM = 12 +) + +// enum uverbs_attrs_destroy_cq_cmd_attr_ids. +const ( + UVERBS_ATTR_DESTROY_CQ_HANDLE = 0 + UVERBS_ATTR_DESTROY_CQ_RESP = 1 +) + +// enum uverbs_attrs_create_qp_cmd_attr_ids. +const ( + UVERBS_ATTR_CREATE_QP_HANDLE = 0 + UVERBS_ATTR_CREATE_QP_XRCD_HANDLE = 1 + UVERBS_ATTR_CREATE_QP_PD_HANDLE = 2 + UVERBS_ATTR_CREATE_QP_SRQ_HANDLE = 3 + UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE = 4 + UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE = 5 + UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE = 6 + UVERBS_ATTR_CREATE_QP_USER_HANDLE = 7 + UVERBS_ATTR_CREATE_QP_CAP = 8 + UVERBS_ATTR_CREATE_QP_TYPE = 9 + UVERBS_ATTR_CREATE_QP_FLAGS = 10 + UVERBS_ATTR_CREATE_QP_SOURCE_QPN = 11 + UVERBS_ATTR_CREATE_QP_EVENT_FD = 12 + UVERBS_ATTR_CREATE_QP_RESP_CAP = 13 + UVERBS_ATTR_CREATE_QP_RESP_QP_NUM = 14 + UVERBS_ATTR_CREATE_QP_BUF_UMEM = 15 + UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM = 16 + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM = 17 +) + +// enum uverbs_attrs_destroy_qp_cmd_attr_ids. +const ( + UVERBS_ATTR_DESTROY_QP_HANDLE = 0 + UVERBS_ATTR_DESTROY_QP_RESP = 1 +) + +// enum uverbs_attrs_async_event_create. +const UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE = 0 + +// Driver-private attribute IDs (>= UVERBS_ID_DRIVER_NS) shared across +// methods. UHW_IN/UHW_OUT carry vendor driver data (e.g. mlx5's +// mlx5_ib_create_cq / mlx5_ib_create_qp with the CQ/QP DMA buffer pointers). +const ( + UVERBS_ATTR_UHW_IN = 0x1000 + UVERBS_ATTR_UHW_OUT = 0x1001 +) + +// mlx5 driver-namespace object/method/attr IDs from +// include/uapi/rdma/mlx5_user_ioctl_cmds.h. The UAR (User Access Region) is +// the CQ/QP doorbell page. +const ( + MLX5_IB_OBJECT_UAR = 0x1008 + + // enum mlx5_ib_uar_obj_methods. + MLX5_IB_METHOD_UAR_OBJ_ALLOC = 0x1000 + MLX5_IB_METHOD_UAR_OBJ_DESTROY = 0x1001 + + // enum mlx5_ib_uar_obj_alloc_attrs. + MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE = 0x1000 + MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE = 0x1001 + MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET = 0x1002 + MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH = 0x1003 + MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID = 0x1004 + + // enum mlx5_ib_uar_obj_destroy_attrs. + MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE = 0x1000 +) + +// EFA driver-namespace method/attr IDs from include/uapi/rdma/efa-abi.h. EFA +// extends the standard UVERBS_OBJECT_MR with a query method returning the +// interconnect IDs an RDMA-read/write source MR must advertise to peers. +const ( + // enum efa_mr_methods. + EFA_IB_METHOD_MR_QUERY = 0x1000 + + // enum efa_query_mr_attrs. + EFA_IB_ATTR_QUERY_MR_HANDLE = 0x1000 + EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY = 0x1001 + EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID = 0x1002 + EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID = 0x1003 + EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID = 0x1004 +) + +// Legacy write(2)-path command numbers (enum ib_uverbs_write_cmds, +// include/uapi/rdma/ib_user_verbs.h), as carried by the INVOKE_WRITE +// WRITE_CMD attribute. +const ( + IB_USER_VERBS_CMD_REG_MR = 9 + IB_USER_VERBS_CMD_DEREG_MR = 13 +) + +// UverbsRegMR is struct ib_uverbs_reg_mr, the legacy write-path REG_MR +// command (include/uapi/rdma/ib_user_verbs.h). Variable-length driver data +// may follow. +// +// +marshal +type UverbsRegMR struct { + _ structs.HostLayout + Response uint64 + Start uint64 + Length uint64 + HcaVA uint64 + PDHandle uint32 + AccessFlags uint32 +} + +// UverbsRegMRResp is struct ib_uverbs_reg_mr_resp, REG_MR's response +// (include/uapi/rdma/ib_user_verbs.h). +// +// +marshal +type UverbsRegMRResp struct { + _ structs.HostLayout + MRHandle uint32 + LKey uint32 + RKey uint32 +} + +// Mlx5CreatePrefix is the common prefix of struct mlx5_ib_create_cq and +// struct mlx5_ib_create_qp (include/uapi/rdma/mlx5-abi.h): the work-queue +// buffer and doorbell guest addresses. +// +// +marshal +type Mlx5CreatePrefix struct { + _ structs.HostLayout + BufAddr uint64 + DBAddr uint64 +} diff --git a/pkg/abi/nvgpu/frontend.go b/pkg/abi/nvgpu/frontend.go index bc4db0c6493..4ffba22b39b 100644 --- a/pkg/abi/nvgpu/frontend.go +++ b/pkg/abi/nvgpu/frontend.go @@ -28,15 +28,16 @@ const NV_IOCTL_MAGIC = uint32('F') // Note that these are only the IOC_NR part of the ioctl command. const ( // From kernel-open/common/inc/nv-ioctl-numbers.h: - NV_IOCTL_BASE = 200 - NV_ESC_CARD_INFO = NV_IOCTL_BASE + 0 - NV_ESC_REGISTER_FD = NV_IOCTL_BASE + 1 - NV_ESC_ALLOC_OS_EVENT = NV_IOCTL_BASE + 6 - NV_ESC_FREE_OS_EVENT = NV_IOCTL_BASE + 7 - NV_ESC_CHECK_VERSION_STR = NV_IOCTL_BASE + 10 - NV_ESC_ATTACH_GPUS_TO_FD = NV_IOCTL_BASE + 12 - NV_ESC_SYS_PARAMS = NV_IOCTL_BASE + 14 - NV_ESC_WAIT_OPEN_COMPLETE = NV_IOCTL_BASE + 18 + NV_IOCTL_BASE = 200 + NV_ESC_CARD_INFO = NV_IOCTL_BASE + 0 + NV_ESC_REGISTER_FD = NV_IOCTL_BASE + 1 + NV_ESC_ALLOC_OS_EVENT = NV_IOCTL_BASE + 6 + NV_ESC_FREE_OS_EVENT = NV_IOCTL_BASE + 7 + NV_ESC_CHECK_VERSION_STR = NV_IOCTL_BASE + 10 + NV_ESC_ATTACH_GPUS_TO_FD = NV_IOCTL_BASE + 12 + NV_ESC_SYS_PARAMS = NV_IOCTL_BASE + 14 + NV_ESC_EXPORT_TO_DMABUF_FD = NV_IOCTL_BASE + 17 + NV_ESC_WAIT_OPEN_COMPLETE = NV_IOCTL_BASE + 18 // From kernel-open/common/inc/nv-ioctl-numa.h: NV_ESC_NUMA_INFO = NV_IOCTL_BASE + 15 @@ -203,6 +204,114 @@ func (p *IoctlWaitOpenComplete) SetStatus(status uint32) { p.AdapterStatus = status } +// NV_DMABUF_EXPORT_MAX_HANDLES is the fixed size of the handle/offset/size +// arrays in nv_ioctl_export_to_dma_buf_fd_t. +// From kernel-open/common/inc/nv-ioctl.h. +const NV_DMABUF_EXPORT_MAX_HANDLES = 128 + +// IoctlExportToDMABufFD is nv_ioctl_export_to_dma_buf_fd_t, the parameter type +// for NV_ESC_EXPORT_TO_DMABUF_FD (kernel-open/common/inc/nv-ioctl.h). +// +// +marshal +type IoctlExportToDMABufFD struct { + _ structs.HostLayout + FD int32 + HClient Handle + TotalObjects uint32 + NumObjects uint32 + Index uint32 + Pad0 uint32 + TotalSize uint64 + Handles [NV_DMABUF_EXPORT_MAX_HANDLES]Handle + Offsets [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Sizes [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Status uint32 + Pad1 uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlExportToDMABufFD) GetFrontendFD() int32 { return p.FD } + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlExportToDMABufFD) SetFrontendFD(fd int32) { p.FD = fd } + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlExportToDMABufFD) GetStatus() uint32 { return p.Status } + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlExportToDMABufFD) SetStatus(status uint32) { p.Status = status } + +// IoctlExportToDMABufFD_V570 is the updated version of +// nv_ioctl_export_to_dma_buf_fd_t since 570.86.15. +// +// +marshal +type IoctlExportToDMABufFD_V570 struct { + _ structs.HostLayout + FD int32 + HClient Handle + TotalObjects uint32 + NumObjects uint32 + Index uint32 + Pad0 uint32 + TotalSize uint64 + MappingType uint8 + Pad1 [3]byte + Handles [NV_DMABUF_EXPORT_MAX_HANDLES]Handle + Pad2 uint32 + Offsets [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Sizes [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Status uint32 + Pad3 uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlExportToDMABufFD_V570) GetFrontendFD() int32 { return p.FD } + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlExportToDMABufFD_V570) SetFrontendFD(fd int32) { p.FD = fd } + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlExportToDMABufFD_V570) GetStatus() uint32 { return p.Status } + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlExportToDMABufFD_V570) SetStatus(status uint32) { p.Status = status } + +// IoctlExportToDMABufFD_V580 is the updated version of +// nv_ioctl_export_to_dma_buf_fd_t since 580.65.06. +// +// +marshal +type IoctlExportToDMABufFD_V580 struct { + _ structs.HostLayout + FD int32 + HClient Handle + TotalObjects uint32 + NumObjects uint32 + Index uint32 + Pad0 uint32 + TotalSize uint64 + MappingType uint8 + AllowMmap uint8 + Pad1 [2]byte + Handles [NV_DMABUF_EXPORT_MAX_HANDLES]Handle + Pad2 uint32 + Offsets [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Sizes [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Status uint32 + Pad3 uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlExportToDMABufFD_V580) GetFrontendFD() int32 { return p.FD } + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlExportToDMABufFD_V580) SetFrontendFD(fd int32) { p.FD = fd } + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlExportToDMABufFD_V580) GetStatus() uint32 { return p.Status } + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlExportToDMABufFD_V580) SetStatus(status uint32) { p.Status = status } + // IoctlNVOS02ParametersWithFD is the parameter type for NV_ESC_RM_ALLOC_MEMORY. // // +marshal diff --git a/pkg/rdma/collect.go b/pkg/rdma/collect.go index 997324e41e1..ec0c3aabb7d 100644 --- a/pkg/rdma/collect.go +++ b/pkg/rdma/collect.go @@ -55,7 +55,7 @@ var ibAttrNames = []string{ // table repopulates when netdevs move namespaces and acquire addresses; // link state and rate can change on retrain). var portLiveAttrNames = []string{ - "state", "phys_state", "rate", "lid", "sm_lid", "sm_sl", + "state", "phys_state", "rate", "lid", "lid_mask_count", "sm_lid", "sm_sl", } // Per-port attributes that are fixed for the sandbox lifetime. @@ -182,13 +182,18 @@ func Collect(sysRoot string, uverbs []UverbsSpec) (*Snapshot, error) { } } - // Materialize every PCI node with its static attributes. + // Materialize every PCI node with its static attributes and config space. for p := range pciPaths { attrs, err := readAttrs(path.Join(sysRoot, p), pciAttrNames) if err != nil { return nil, fmt.Errorf("PCI node %q: %w", p, err) } - s.PCINodes = append(s.PCINodes, PCINode{Path: p, Attrs: attrs}) + // config is best-effort: root complexes and some bridges lack it. + config, err := os.ReadFile(path.Join(sysRoot, p, "config")) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("reading PCI config of %q: %w", p, err) + } + s.PCINodes = append(s.PCINodes, PCINode{Path: p, Attrs: attrs, Config: config}) } sort.Slice(s.PCINodes, func(i, j int) bool { return s.PCINodes[i].Path < s.PCINodes[j].Path }) diff --git a/pkg/rdma/snapshot.go b/pkg/rdma/snapshot.go index 193304a9d01..79b62c4a0eb 100644 --- a/pkg/rdma/snapshot.go +++ b/pkg/rdma/snapshot.go @@ -40,6 +40,11 @@ type PCINode struct { // Attrs maps attribute file name to contents (verbatim, including any // trailing newline). Attrs map[string]string `json:"attrs"` + // Config is the raw PCI config space ("config" file), or nil if absent. + // hwloc (used by aws-ofi-nccl for NCCL topology) reads it to recover the + // PCI-bridge bus-number registers and PCIe link attributes; without it + // hwloc cannot reconstruct the bridge hierarchy. + Config []byte `json:"config,omitempty"` } // Port is the per-IB-port state. Attributes split into static (immutable diff --git a/pkg/sentry/devices/nvproxy/frontend.go b/pkg/sentry/devices/nvproxy/frontend.go index 83a28a55723..1b4da287472 100644 --- a/pkg/sentry/devices/nvproxy/frontend.go +++ b/pkg/sentry/devices/nvproxy/frontend.go @@ -369,6 +369,125 @@ func frontendRegisterFD(fi *frontendIoctlState) (uintptr, error) { return frontendIoctlInvokeNoStatus(fi, &ioctlParams) } +// frontendExportToDMABufFD handles NV_ESC_EXPORT_TO_DMABUF_FD. CUDA uses this +// to export GPU memory as a dma-buf fd for ibv_reg_dmabuf_mr (GPUDirect RDMA). +func frontendExportToDMABufFD[Params any, PtrParams hasFrontendFDAndStatusPtr[Params]](fi *frontendIoctlState) (uintptr, error) { + var ioctlParamsValue Params + ioctlParams := PtrParams(&ioctlParamsValue) + if int(fi.ioctlParamsSize) != ioctlParams.SizeBytes() { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + + // The FD field is both input and output (see nv_dma_buf_export() in + // kernel-open/nvidia/nv-dmabuf.c): + // - FD == -1: the driver creates a new dma-buf and returns its fd by + // updating the ioctlParams. We wrap that host fd in a new application + // FD installed in the task's fd table and return it via ioctlParams to + // the app, so the app can hand it to the RDMA verbs library. + // - FD >= 0: the app is appending more handles to a dma-buf it exported + // in an earlier call, identified by that fd. The value is an app FD, + // we translate it back to the host fd of the dma-buf we wrapped + // before. The driver leaves the host fd unchanged, so we restore the + // app fd on the way out. + if appFD := ioctlParams.GetFrontendFD(); appFD >= 0 { + fileGeneric, _ := fi.t.FDTable().Get(appFD) + if fileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer fileGeneric.DecRef(fi.ctx) + wrapper, ok := fileGeneric.Impl().(*dmaBufFDWrapper) + if !ok { + return 0, linuxerr.EINVAL + } + ioctlParams.SetFrontendFD(wrapper.hostFD) + n, err := frontendIoctlInvokeNoStatus(fi, ioctlParams) + ioctlParams.SetFrontendFD(appFD) + if err != nil { + return n, err + } + if _, err := ioctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil + } + + // Create path: FD == -1 (any other negative value is rejected by the + // driver, so FD < -1 will fail below). + n, err := frontendIoctlInvokeNoStatus(fi, ioctlParams) + if err != nil { + return n, err + } + + // The driver only allocates a dma-buf fd when the export succeeded. + if ioctlParams.GetStatus() == nvgpu.NV_OK && ioctlParams.GetFrontendFD() >= 0 { + sandboxFD, err := newDMABufFDWrapper(fi.t, ioctlParams.GetFrontendFD()) + if err != nil { + return n, err + } + ioctlParams.SetFrontendFD(sandboxFD) + if _, err := ioctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + // Roll back the fd we installed so the guest doesn't retain a + // dma-buf it never learned the number of. Remove drops the table's + // reference; DecRef releases it (closing the host fd). + if removed := fi.t.FDTable().Remove(fi.ctx, sandboxFD); removed != nil { + removed.DecRef(fi.ctx) + } + return n, err + } + return n, nil + } + + if _, err := ioctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil +} + +// dmaBufFDWrapper wraps a host dma-buf fd returned by the NVIDIA driver's +// export operation as an application FD. It implements vfs.HostFDProvider so +// that rdmaproxy can recover the host fd to hand to ibv_reg_dmabuf_mr. +// +// +stateify savable +type dmaBufFDWrapper struct { + vfsfd vfs.FileDescription + vfs.FileDescriptionDefaultImpl + vfs.DentryMetadataFileDescriptionImpl + vfs.NoLockFD + + hostFD int32 +} + +// HostFD implements vfs.HostFDProvider.HostFD. +func (fd *dmaBufFDWrapper) HostFD() int { + return int(fd.hostFD) +} + +// Release implements vfs.FileDescriptionImpl.Release. +func (fd *dmaBufFDWrapper) Release(ctx context.Context) { + unix.Close(int(fd.hostFD)) +} + +// newDMABufFDWrapper takes ownership of hostFD, wraps it in an application +// FileDescription and installs it in t's FD table. It returns the application +// FD number. +func newDMABufFDWrapper(t *kernel.Task, hostFD int32) (int32, error) { + vfsObj := t.Kernel().VFS() + vd := vfsObj.NewAnonVirtualDentry("[nvidia-dmabuf]") + defer vd.DecRef(t) + w := &dmaBufFDWrapper{hostFD: hostFD} + if err := w.vfsfd.Init(w, linux.O_RDWR, t.Credentials(), vd.Mount(), vd.Dentry(), &vfs.FileDescriptionOptions{ + UseDentryMetadata: true, + }); err != nil { + unix.Close(int(hostFD)) + return -1, err + } + defer w.vfsfd.DecRef(t) + return t.NewFDFrom(0, &w.vfsfd, kernel.FDFlags{CloseOnExec: true}) +} + func frontendIoctlHasFD[Params any, PtrParams hasFrontendFDAndStatusPtr[Params]](fi *frontendIoctlState) (uintptr, error) { var ioctlParamsValue Params ioctlParams := PtrParams(&ioctlParamsValue) diff --git a/pkg/sentry/devices/nvproxy/nvconf/caps.go b/pkg/sentry/devices/nvproxy/nvconf/caps.go index 6fc2cf78ef0..a4fbfc89d48 100644 --- a/pkg/sentry/devices/nvproxy/nvconf/caps.go +++ b/pkg/sentry/devices/nvproxy/nvconf/caps.go @@ -41,6 +41,7 @@ const ( // requiring to be enabled explicitly. CapFabricIMEXManagement // NV_RM_CAP_SYS_FABRIC_IMEX_MGMT CapProfiling // GPU hardware performance counter access (Nsight Compute/Systems) + CapRDMA // GPUDirect RDMA: exporting GPU memory to a dma-buf fd numValidCaps int = iota ) @@ -55,7 +56,7 @@ const ( // SupportedDriverCaps is the set of driver capabilities that are supported by // nvproxy. - SupportedDriverCaps = AllContainerDriverCaps | CapFabricIMEXManagement | CapProfiling + SupportedDriverCaps = AllContainerDriverCaps | CapFabricIMEXManagement | CapProfiling | CapRDMA // AllContainerDriverCaps is the subset of SupportedDriverCaps that are // enabled when enabling "all" capabilities is requested, which excludes @@ -91,6 +92,8 @@ func (c DriverCaps) individualString() string { return "fabric-imex-mgmt" case CapProfiling: return "profiling" + case CapRDMA: + return "rdma" default: panic(fmt.Sprintf("capability has no string mapping: %x", uint16(c))) } diff --git a/pkg/sentry/devices/nvproxy/seccomp_filters.go b/pkg/sentry/devices/nvproxy/seccomp_filters.go index f5173272c33..b677f3dd3e1 100644 --- a/pkg/sentry/devices/nvproxy/seccomp_filters.go +++ b/pkg/sentry/devices/nvproxy/seccomp_filters.go @@ -49,6 +49,7 @@ func frontendIoctlFilters(enabledCaps nvconf.DriverCaps) []seccomp.SyscallRule { {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_FREE_OS_EVENT, nvgpu.SizeofIoctlFreeOSEvent)), compUtil}, {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_SYS_PARAMS, nvgpu.SizeofIoctlSysParams)), compUtil}, {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_WAIT_OPEN_COMPLETE, nvgpu.SizeofIoctlWaitOpenComplete)), compUtil}, + {seccomp.MaskedEqual(notIocSizeMask, frontendIoctlCmd(nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD, 0)), nvconf.CapRDMA}, {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC_MEMORY, nvgpu.SizeofIoctlNVOS02ParametersWithFD)), compUtil | nvconf.CapGraphics}, {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_FREE, nvgpu.SizeofNVOS00Parameters)), compUtil}, {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_CONTROL, nvgpu.SizeofNVOS54Parameters)), compUtil}, diff --git a/pkg/sentry/devices/nvproxy/version.go b/pkg/sentry/devices/nvproxy/version.go index 229fa37f08f..9b36a9129f8 100644 --- a/pkg/sentry/devices/nvproxy/version.go +++ b/pkg/sentry/devices/nvproxy/version.go @@ -175,6 +175,7 @@ func Init() { nvgpu.NV_ESC_ALLOC_OS_EVENT: feHandler(frontendIoctlHasFD[nvgpu.IoctlAllocOSEvent], compUtil), nvgpu.NV_ESC_FREE_OS_EVENT: feHandler(frontendIoctlHasFD[nvgpu.IoctlFreeOSEvent], compUtil), nvgpu.NV_ESC_NUMA_INFO: feHandler(rmNumaInfo, compUtil), + nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD: feHandler(frontendExportToDMABufFD[nvgpu.IoctlExportToDMABufFD], nvconf.CapRDMA), nvgpu.NV_ESC_RM_ALLOC_CONTEXT_DMA2: feHandler(rmAllocContextDMA2, nvconf.CapGraphics), nvgpu.NV_ESC_RM_ALLOC_MEMORY: feHandler(rmAllocMemory, compUtil|nvconf.CapGraphics), nvgpu.NV_ESC_RM_FREE: feHandler(rmFree, compUtil), @@ -490,6 +491,7 @@ func Init() { nvgpu.NV_ESC_ALLOC_OS_EVENT: ioctlInfoWithStructName("NV_ESC_ALLOC_OS_EVENT", nvgpu.IoctlAllocOSEvent{}, "nv_ioctl_alloc_os_event_t"), nvgpu.NV_ESC_FREE_OS_EVENT: ioctlInfoWithStructName("NV_ESC_FREE_OS_EVENT", nvgpu.IoctlFreeOSEvent{}, "nv_ioctl_free_os_event_t"), nvgpu.NV_ESC_NUMA_INFO: simpleIoctlInfo("NV_ESC_NUMA_INFO"), // No params struct because nvproxy ignores this ioctl + nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD: ioctlInfoWithStructName("NV_ESC_EXPORT_TO_DMABUF_FD", nvgpu.IoctlExportToDMABufFD{}, "nv_ioctl_export_to_dma_buf_fd_t"), nvgpu.NV_ESC_RM_ALLOC_MEMORY: ioctlInfoWithStructName("NV_ESC_RM_ALLOC_MEMORY", nvgpu.IoctlNVOS02ParametersWithFD{}, "nv_ioctl_nvos02_parameters_with_fd"), nvgpu.NV_ESC_RM_FREE: ioctlInfo("NV_ESC_RM_FREE", nvgpu.NVOS00_PARAMETERS{}), nvgpu.NV_ESC_RM_CONTROL: ioctlInfo("NV_ESC_RM_CONTROL", nvgpu.NVOS54_PARAMETERS{}), @@ -1005,9 +1007,11 @@ func Init() { abi.allocationClass[nvgpu.BLACKWELL_COMPUTE_B] = allocHandler(rmAllocSimple[nvgpu.NV_GR_ALLOCATION_PARAMETERS], compUtil) abi.allocationClass[nvgpu.BLACKWELL_USERMODE_A] = allocHandler(rmAllocSimple[nvgpu.NV_HOPPER_USERMODE_A_PARAMS], compUtil) abi.allocationClass[nvgpu.NVCFB7_VIDEO_ENCODER] = allocHandler(rmAllocSimple[nvgpu.NV_MSENC_ALLOCATION_PARAMETERS], nvconf.CapVideo) + abi.frontendIoctl[nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD] = feHandler(frontendExportToDMABufFD[nvgpu.IoctlExportToDMABufFD_V570], nvconf.CapRDMA) prevGetInfo := abi.getInfo abi.getInfo = func() *DriverABIInfo { info := prevGetInfo() + info.FrontendInfos[nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD] = ioctlInfoWithStructName("NV_ESC_EXPORT_TO_DMABUF_FD", nvgpu.IoctlExportToDMABufFD_V570{}, "nv_ioctl_export_to_dma_buf_fd_t") info.ControlInfos[nvgpu.NVB0CC_CTRL_CMD_RESERVE_CCU_PROF] = simpleIoctlInfo("NVB0CC_CTRL_CMD_RESERVE_CCU_PROF", "NVB0CC_CTRL_RESERVE_CCUPROF_PARAMS") info.ControlInfos[nvgpu.NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT] = simpleIoctlInfo("NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT", "NV2080_CTRL_FB_DRAM_ENCRYPTION_INFOROM_SUPPORT_PARAMS") info.ControlInfos[nvgpu.NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS] = simpleIoctlInfo("NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS", "NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_STATUS_PARAMS") @@ -1064,10 +1068,12 @@ func Init() { abi.allocationClass[nvgpu.NVD1B7_VIDEO_ENCODER] = allocHandler(rmAllocSimple[nvgpu.NV_MSENC_ALLOCATION_PARAMETERS], nvconf.CapVideo) abi.controlCmd[nvgpu.NV2080_CTRL_CMD_GPU_GET_SKYLINE_INFO] = ctrlHandler(rmControlSimple, compUtil|nvconf.CapGraphics) abi.controlCmd[nvgpu.NV2080_CTRL_CMD_ECC_GET_REPAIR_STATUS] = ctrlHandler(rmControlSimple, nvconf.CapGraphics) + abi.frontendIoctl[nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD] = feHandler(frontendExportToDMABufFD[nvgpu.IoctlExportToDMABufFD_V580], nvconf.CapRDMA) prevGetInfo := abi.getInfo abi.getInfo = func() *DriverABIInfo { info := prevGetInfo() + info.FrontendInfos[nvgpu.NV_ESC_EXPORT_TO_DMABUF_FD] = ioctlInfoWithStructName("NV_ESC_EXPORT_TO_DMABUF_FD", nvgpu.IoctlExportToDMABufFD_V580{}, "nv_ioctl_export_to_dma_buf_fd_t") info.FrontendInfos[nvgpu.NV_ESC_RM_MAP_MEMORY_DMA] = ioctlInfoWithStructName("NV_ESC_RM_MAP_MEMORY_DMA", nvgpu.NVOS46_PARAMETERS_V580{}, "NVOS46_PARAMETERS") info.AllocationInfos[nvgpu.FERMI_VASPACE_A] = ioctlInfoWithStructName("FERMI_VASPACE_A", nvgpu.NV_VASPACE_ALLOCATION_PARAMETERS_V580{}, "NV_VASPACE_ALLOCATION_PARAMETERS") info.AllocationInfos[nvgpu.NVCEB7_VIDEO_ENCODER] = ioctlInfo("NVCEB7_VIDEO_ENCODER", nvgpu.NV_MSENC_ALLOCATION_PARAMETERS{}) diff --git a/pkg/sentry/devices/rdmaproxy/BUILD b/pkg/sentry/devices/rdmaproxy/BUILD new file mode 100644 index 00000000000..082cd010268 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/BUILD @@ -0,0 +1,45 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "rdmaproxy", + srcs = [ + "driver.go", + "ioctl.go", + "ioctl_unsafe.go", + "rdmaproxy.go", + "schema.go", + "seccomp_filter.go", + ], + visibility = [ + "//pkg/sentry:internal", + "//runsc:__subpackages__", + ], + deps = [ + "//pkg/abi/ib", + "//pkg/atomicbitops", + "//pkg/abi/linux", + "//pkg/cleanup", + "//pkg/context", + "//pkg/devutil", + "//pkg/errors/linuxerr", + "//pkg/fdnotifier", + "//pkg/hostarch", + "//pkg/log", + "//pkg/seccomp", + "//pkg/sentry/arch", + "//pkg/sentry/fsutil", + "//pkg/sentry/kernel", + "//pkg/sentry/kernel/auth", + "//pkg/sentry/memmap", + "//pkg/sentry/mm", + "//pkg/sentry/vfs", + "//pkg/usermem", + "//pkg/waiter", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/pkg/sentry/devices/rdmaproxy/cxproxy/BUILD b/pkg/sentry/devices/rdmaproxy/cxproxy/BUILD new file mode 100644 index 00000000000..719da2c2b97 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/cxproxy/BUILD @@ -0,0 +1,24 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "cxproxy", + srcs = [ + "cxproxy.go", + ], + visibility = [ + "//pkg/sentry:internal", + "//runsc:__subpackages__", + ], + deps = [ + "//pkg/abi/ib", + "//pkg/cleanup", + "//pkg/hostarch", + "//pkg/sentry/devices/rdmaproxy", + "//pkg/sentry/kernel", + ], +) diff --git a/pkg/sentry/devices/rdmaproxy/cxproxy/cxproxy.go b/pkg/sentry/devices/rdmaproxy/cxproxy/cxproxy.go new file mode 100644 index 00000000000..13644f7f437 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/cxproxy/cxproxy.go @@ -0,0 +1,116 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cxproxy implements the rdmaproxy.Driver plug-in for Mellanox/NVIDIA +// ConnectX adapters (mlx5_core / mlx5_ib). It mirrors the CQ/QP work-queue and +// doorbell DMA buffers referenced by the mlx5 driver-private UHW payload. +// +// To make this driver available, Init() must be called. +package cxproxy + +import ( + "gvisor.dev/gvisor/pkg/abi/ib" + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy" + "gvisor.dev/gvisor/pkg/sentry/kernel" +) + +// driverName matches the DRIVER= field of +// /sys/class/infiniband//device/uevent for ConnectX adapters bound to +// the upstream mlx5 stack. runsc looks this up via rdmaproxy.LookupDriver and +// attaches the resulting driver to the corresponding uverbs device. +const driverName = "mlx5_core" + +// cxDriver is the rdmaproxy.Driver implementation for ConnectX adapters. +type cxDriver struct{} + +// Name implements rdmaproxy.Driver.Name. +func (cxDriver) Name() string { return driverName } + +// PrepareCreateDMA implements rdmaproxy.Driver.PrepareCreateDMA. It mirrors the +// work-queue buffer (buf_addr) and doorbell page (db_addr) referenced by the +// mlx5 driver-private payload, rewriting both fields to their sentry-side +// mappings before the ioctl is forwarded to the host kernel. +// +// mlx5 supplies no explicit buffer length in the payload, so the work-queue +// buffer is mirrored from buf_addr to the end of its VMA (the buffer is a +// single mlx5-allocated mapping); rdmaproxy caps the pinned length. The +// doorbell is a small record within one page, so a single page is mirrored. +func (cxDriver) PrepareCreateDMA(t *kernel.Task, uhwIn []byte) (*rdmaproxy.PinnedDMABufs, error) { + var prefix ib.Mlx5CreatePrefix + if len(uhwIn) < prefix.SizeBytes() { + return nil, nil + } + prefix.UnmarshalBytes(uhwIn) + + var bufs rdmaproxy.PinnedDMABufs + var cu cleanup.Cleanup + defer cu.Clean() + + if prefix.BufAddr != 0 { + vmaRange, err := t.MemoryManager().FindVMARange(hostarch.Addr(prefix.BufAddr)) + if err != nil { + return nil, err + } + length := uint64(vmaRange.End) - prefix.BufAddr + mp, sentryVA, err := rdmaproxy.MirrorAppPages(t, prefix.BufAddr, length) + if err != nil { + return nil, err + } + bufs.Buf = mp + cu.Add(func() { mp.Release(t) }) + prefix.BufAddr = uint64(sentryVA) + } + + if prefix.DBAddr != 0 { + mp, sentryVA, err := rdmaproxy.MirrorAppPages(t, prefix.DBAddr, hostarch.PageSize) + if err != nil { + return nil, err + } + bufs.DB = mp + cu.Add(func() { mp.Release(t) }) + prefix.DBAddr = uint64(sentryVA) + } + + cu.Release() + prefix.MarshalBytes(uhwIn) + return &bufs, nil +} + +// Schemas implements rdmaproxy.Driver.Schemas. It models the mlx5 UAR (User +// Access Region — the CQ/QP doorbell page) object: alloc returns an mmap +// offset that the app maps through the uverbs FD via the generic passthrough, +// so no address translation is needed here. +func (cxDriver) Schemas() map[uint32]*rdmaproxy.MethodSchema { + return map[uint32]*rdmaproxy.MethodSchema{ + rdmaproxy.SchemaKey(ib.MLX5_IB_OBJECT_UAR, ib.MLX5_IB_METHOD_UAR_OBJ_ALLOC): { + Attrs: map[uint16]rdmaproxy.AttrType{ + ib.MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE: rdmaproxy.AttrIdr, + ib.MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE: rdmaproxy.AttrPtrIn, + ib.MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET: rdmaproxy.AttrPtrOut, + ib.MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH: rdmaproxy.AttrPtrOut, + ib.MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID: rdmaproxy.AttrPtrOut, + }, + }, + rdmaproxy.SchemaKey(ib.MLX5_IB_OBJECT_UAR, ib.MLX5_IB_METHOD_UAR_OBJ_DESTROY): { + Attrs: map[uint16]rdmaproxy.AttrType{ + ib.MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE: rdmaproxy.AttrIdr, + }, + }, + } +} + +// Init registers the ConnectX (mlx5) driver plug-in with the rdmaproxy core. +func Init() { rdmaproxy.RegisterDriver(cxDriver{}) } diff --git a/pkg/sentry/devices/rdmaproxy/driver.go b/pkg/sentry/devices/rdmaproxy/driver.go new file mode 100644 index 00000000000..b0bfc4898d7 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/driver.go @@ -0,0 +1,119 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdmaproxy + +import ( + "sync" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/kernel" +) + +// Driver is the per-vendor plug-in interface for mirroring the DMA buffers a +// CQ or QP CREATE references through its driver-private (UHW) payload. +// +// The vendor-neutral core routes every standard UVERBS object/method from the +// schema, but the work-queue and doorbell buffer pointers embedded in the +// driver-private data are vendor-specific (mlx5_ib_create_cq / mlx5_ib_create_qp, +// efa_ibv_create_qp, ...). The core hands the copied-in UHW_IN payload to the +// matching driver during CQ/QP CREATE. +// +// Implementations live in vendor-specific subpackages, e.g. +// pkg/sentry/devices/rdmaproxy/cxproxy for ConnectX (mlx5). +type Driver interface { + // Name returns the driver name as it appears in the host's + // /sys/class/infiniband//device/uevent DRIVER= field (e.g. + // "mlx5_core"). This is the string passed to Register and used to look + // up the driver at registration time. + Name() string + + // PrepareCreateDMA mirrors the DMA buffers referenced by the vendor + // driver-private payload of a CQ or QP CREATE and rewrites the embedded + // app addresses in uhwIn IN PLACE to the corresponding sentry-side + // mappings. uhwIn is the sentry-side copy of the UHW_IN attribute the + // core already copied in from the guest; the host kernel will forward it + // to the vendor driver verbatim. The CQ and QP driver-private structs + // share the buf/doorbell prefix this needs, so CREATE type is not passed. + // + // It returns a handle tracking the mirrored pages, stored by the core + // against the resulting CQ/QP handle and released on teardown, or nil if + // the payload referenced no buffers. A non-nil error aborts the ioctl. + PrepareCreateDMA(t *kernel.Task, uhwIn []byte) (*PinnedDMABufs, error) + + // Schemas returns the driver-namespace (object, method) pairs this driver + // models (IDs >= UVERBS_ID_DRIVER_NS), keyed by SchemaKey, which the core + // merges into its allowlist. + Schemas() map[uint32]*MethodSchema +} + +// driverRegistry holds Drivers registered by per-vendor packages keyed by +// Driver.Name() (which must match the host PCI driver name). +type driverRegistry struct { + mu sync.RWMutex + drivers map[string]Driver +} + +var registryOnce sync.Once +var registry *driverRegistry + +// driverReg returns the process-wide driver registry. +func driverReg() *driverRegistry { + registryOnce.Do(func() { + registry = &driverRegistry{drivers: make(map[string]Driver)} + }) + return registry +} + +// RegisterDriver makes drv available for lookup by drv.Name(). Duplicate +// registrations under the same name overwrite the previous entry (last writer +// wins) so tests can stub a driver. +func RegisterDriver(drv Driver) { + reg := driverReg() + name := drv.Name() + reg.mu.Lock() + reg.drivers[name] = drv + reg.mu.Unlock() + log.Infof("rdmaproxy: registered driver name=%s", name) +} + +// LookupDriver returns the driver registered under name, or nil if none. +func LookupDriver(name string) Driver { + reg := driverReg() + reg.mu.RLock() + defer reg.mu.RUnlock() + return reg.drivers[name] +} + +// rangeDrivers calls f for each registered driver. +func rangeDrivers(f func(Driver)) { + reg := driverReg() + reg.mu.RLock() + defer reg.mu.RUnlock() + for _, drv := range reg.drivers { + f(drv) + } +} + +// RegisteredDrivers returns the names of all registered drivers. +func RegisteredDrivers() []string { + reg := driverReg() + reg.mu.RLock() + defer reg.mu.RUnlock() + names := make([]string, 0, len(reg.drivers)) + for name := range reg.drivers { + names = append(names, name) + } + return names +} diff --git a/pkg/sentry/devices/rdmaproxy/efaproxy/BUILD b/pkg/sentry/devices/rdmaproxy/efaproxy/BUILD new file mode 100644 index 00000000000..87cd4fd0827 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/efaproxy/BUILD @@ -0,0 +1,22 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "efaproxy", + srcs = [ + "efaproxy.go", + ], + visibility = [ + "//pkg/sentry:internal", + "//runsc:__subpackages__", + ], + deps = [ + "//pkg/abi/ib", + "//pkg/sentry/devices/rdmaproxy", + "//pkg/sentry/kernel", + ], +) diff --git a/pkg/sentry/devices/rdmaproxy/efaproxy/efaproxy.go b/pkg/sentry/devices/rdmaproxy/efaproxy/efaproxy.go new file mode 100644 index 00000000000..3babf5d2251 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/efaproxy/efaproxy.go @@ -0,0 +1,71 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package efaproxy implements the rdmaproxy.Driver plug-in for AWS Elastic +// Fabric Adapter (EFA) devices bound to the host `efa` kernel driver. +// +// To make this driver available, Init() must be called. +package efaproxy + +import ( + "gvisor.dev/gvisor/pkg/abi/ib" + "gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy" + "gvisor.dev/gvisor/pkg/sentry/kernel" +) + +// driverName matches the DRIVER= field of +// /sys/class/infiniband//device/uevent for EFA adapters. runsc looks +// this up via rdmaproxy.LookupDriver and attaches the resulting driver to the +// corresponding uverbs device. +const driverName = "efa" + +// efaDriver is the rdmaproxy.Driver implementation for EFA adapters. +type efaDriver struct{} + +// Name implements rdmaproxy.Driver.Name. +func (efaDriver) Name() string { return driverName } + +// PrepareCreateDMA implements rdmaproxy.Driver.PrepareCreateDMA. EFA CQ/QP +// CREATE command structs (efa_ibv_create_cq / efa_ibv_create_qp) carry no +// userspace buffer pointers. The host kernel allocates the work-queue and +// doorbell memory and hands it back through mmap keys in the CREATE *response* +// (q_mmap_key, rq_mmap_key, sq_db_mmap_key, ...); userspace then mmap()s the +// uverbs FD at those offsets, which the rdmaproxy core already forwards to the +// host FD verbatim. There is thus no app memory to mirror or rewrite at CREATE +// time, so PrepareCreateDMA is a no-op. +func (efaDriver) PrepareCreateDMA(t *kernel.Task, uhwIn []byte) (*rdmaproxy.PinnedDMABufs, error) { + return nil, nil +} + +// Schemas implements rdmaproxy.Driver.Schemas. EFA extends the standard MR +// object with a query method returning the interconnect IDs a source MR must +// advertise for RDMA read/write; libfabric's EFA provider issues it during +// endpoint setup. All attributes are handles or fixed scalars, so no address +// translation is needed. +func (efaDriver) Schemas() map[uint32]*rdmaproxy.MethodSchema { + return map[uint32]*rdmaproxy.MethodSchema{ + rdmaproxy.SchemaKey(ib.UVERBS_OBJECT_MR, ib.EFA_IB_METHOD_MR_QUERY): { + Attrs: map[uint16]rdmaproxy.AttrType{ + ib.EFA_IB_ATTR_QUERY_MR_HANDLE: rdmaproxy.AttrIdr, + ib.EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY: rdmaproxy.AttrPtrOut, + ib.EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID: rdmaproxy.AttrPtrOut, + ib.EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID: rdmaproxy.AttrPtrOut, + ib.EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID: rdmaproxy.AttrPtrOut, + }, + }, + } +} + +// Init registers the EFA driver plug-in with the rdmaproxy core. +func Init() { rdmaproxy.RegisterDriver(efaDriver{}) } diff --git a/pkg/sentry/devices/rdmaproxy/ioctl.go b/pkg/sentry/devices/rdmaproxy/ioctl.go new file mode 100644 index 00000000000..473a55dc3e2 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/ioctl.go @@ -0,0 +1,606 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdmaproxy + +import ( + "fmt" + "runtime" + + "gvisor.dev/gvisor/pkg/abi/ib" + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/arch" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/usermem" +) + +// maxNumAttrs bounds the attribute count. No modeled method carries more than +// ~20 attributes; the cap keeps a malicious header from driving large +// allocations and guarantees the whole request fits in a page. +const maxNumAttrs = 64 + +// xlat records one attribute the proxy rewrote in the forwarded request, so the +// original value can be restored (and any output copied back) after the call. +type xlat struct { + // attr points into the request's attrs slice. + attr *ib.UverbsAttr + // orig is the original guest value of the data field, restored before the + // buffer is copied back so the guest never observes a sentry pointer, + // host fd, or relocated address. + orig uint64 + // outBuf, when non-nil, is a sentry PTR_OUT staging buffer whose contents + // are copied back to the guest pointer (orig) after a successful call. + outBuf []byte +} + +// Ioctl implements vfs.FileDescriptionImpl.Ioctl. +func (fd *uverbsFD) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) { + cmd := args[1].Uint() + argPtr := args[2].Pointer() + + t := kernel.TaskFromContext(ctx) + if t == nil { + log.Warningf("rdmaproxy: ioctl called without task context") + return 0, linuxerr.EINVAL + } + + if cmd != ib.RDMAVerbsIoctl { + log.Warningf("rdmaproxy: unsupported ioctl cmd=%#x", cmd) + return 0, linuxerr.ENOTTY + } + return fd.handleRDMAVerbsIoctl(t, argPtr) +} + +// handleRDMAVerbsIoctl translates and forwards a single RDMA_VERBS_IOCTL +// request. The request is validated against the schema; every attribute is +// translated strictly by its declared type; DMA buffers are mirrored; the +// request is forwarded to the host uverbs FD; and outputs are copied back. +func (fd *uverbsFD) handleRDMAVerbsIoctl(t *kernel.Task, argPtr hostarch.Addr) (uintptr, error) { + var hdr ib.UverbsIoctlHdr + if _, err := hdr.CopyIn(t, argPtr); err != nil { + return 0, err + } + + // Validate length using int arithmetic to avoid the u16 overflow that + // would let a large NumAttrs wrap back to a small "valid" length. + if hdr.NumAttrs > maxNumAttrs { + log.Warningf("rdmaproxy: too many attrs: %d (max %d)", hdr.NumAttrs, maxNumAttrs) + return 0, linuxerr.EINVAL + } + expectedLen := int(ib.SizeofUverbsIoctlHdr) + int(hdr.NumAttrs)*int(ib.SizeofUverbsAttr) + if int(hdr.Length) != expectedLen { + log.Warningf("rdmaproxy: bad ioctl length=%d expected=%d (numAttrs=%d)", hdr.Length, expectedLen, hdr.NumAttrs) + return 0, linuxerr.EINVAL + } + + schema := lookupSchema(hdr.ObjectID, hdr.MethodID) + if schema == nil { + log.Warningf("rdmaproxy: unsupported ioctl object=%d method=%d", hdr.ObjectID, hdr.MethodID) + return 0, linuxerr.EINVAL + } + + attrsAddr := argPtr + hostarch.Addr(ib.SizeofUverbsIoctlHdr) + attrs := make([]ib.UverbsAttr, hdr.NumAttrs) + if _, err := ib.CopyUverbsAttrSliceIn(t, attrsAddr, attrs); err != nil { + return 0, err + } + + // staged maps a staged (len>8) attribute's ID to its sentry buffer: a + // runtime.KeepAlive pins them across the syscall, and DMA handlers recover + // a buffer by attr ID. + staged := make(map[uint16][]byte) + // xlats records every rewritten data field for restore / copy-back. + var xlats []xlat + + // Translate each attribute strictly by its schema-declared type. + for i := range attrs { + a := &attrs[i] + typ, ok := schema.Attrs[a.AttrID] + if !ok { + log.Warningf("rdmaproxy: unsupported attr id=%#x on object=%d method=%d", a.AttrID, hdr.ObjectID, hdr.MethodID) + return 0, linuxerr.EINVAL + } + + switch typ { + case AttrPtrIn: + // Inline when len<=8 (forward untouched); otherwise a guest + // pointer whose contents are staged in the sentry. + if a.Len > 8 { + sb, err := fd.copyInPtr(t, a.Data, a.Len) + if err != nil { + return 0, err + } + xlats = append(xlats, xlat{attr: a, orig: a.Data}) + a.Data = sentryDataPtr(sb) + staged[a.AttrID] = sb + } + + case AttrPtrOut: + // Always a guest pointer; stage an output buffer of the declared + // length and copy it back after the call. + if a.Len == 0 { + // No output buffer: forward a null pointer rather than the + // guest's raw data value, restored on copy-out. + xlats = append(xlats, xlat{attr: a, orig: a.Data}) + a.Data = 0 + break + } + // Note that Len is uint16, so the slice allocation below is + // bounded to 64 KiB. + sb := make([]byte, a.Len) + xlats = append(xlats, xlat{attr: a, orig: a.Data, outBuf: sb}) + a.Data = sentryDataPtr(sb) + staged[a.AttrID] = sb + + case AttrInline: + // Left in place for a method-specific handler; must be inline so + // the handler reads the real value, not a staged sentry pointer. + if a.Len > 8 { + return 0, linuxerr.EINVAL + } + + case AttrIdr: + // Object handle: len must be 0, data passed through unchanged. + // For UVERBS_ACCESS_NEW the kernel writes the fresh handle into + // this field of the forwarded buffer; it reaches the guest via + // the copy-out below. + if a.Len != 0 { + return 0, linuxerr.EINVAL + } + + case AttrFdIn: + if a.Len != 0 { + return 0, linuxerr.EINVAL + } + hostFD, file, err := fd.translateInputFD(t, int32(a.Data)) + if err != nil { + return 0, err + } + defer file.DecRef(t) + xlats = append(xlats, xlat{attr: a, orig: a.Data}) + a.Data = uint64(uint32(hostFD)) + + case AttrFdNew: + // Output fd: the kernel installs a host fd in the sentry and + // writes its number here; wrapped after a successful call. + if a.Len != 0 { + return 0, linuxerr.EINVAL + } + + case AttrRawFd: + if a.Len != 0 { + return 0, linuxerr.EINVAL + } + // data_s64; negative means "no fd", forwarded unchanged. + if int64(a.Data) >= 0 { + hostFD, file, err := fd.translateHostBackedFD(t, int32(a.Data)) + if err != nil { + return 0, err + } + defer file.DecRef(t) + xlats = append(xlats, xlat{attr: a, orig: a.Data}) + a.Data = uint64(uint32(hostFD)) + } + + case AttrUnsupported: + log.Warningf("rdmaproxy: unsupported attr id=%#x on object=%d method=%d", a.AttrID, hdr.ObjectID, hdr.MethodID) + return 0, linuxerr.EINVAL + + default: + log.Warningf("rdmaproxy: unsupported attr type %d for id=%#x on object=%d method=%d", typ, a.AttrID, hdr.ObjectID, hdr.MethodID) + return 0, linuxerr.EINVAL + } + } + + // Method-specific DMA / fd pre-processing that operates on inline values + // (which the generic loop leaves untouched). + var mrMirror *MirroredPages + var cqqpMirror *PinnedDMABufs + var dmaCleanup cleanup.Cleanup + defer dmaCleanup.Clean() + + switch schema.Dma { + case DmaMRReg: + mp, err := fd.prepareMRReg(t, attrs, &xlats) + if err != nil { + log.Warningf("rdmaproxy: REG_MR page mirroring: %v", err) + return 0, err + } + if mp != nil { + mrMirror = mp + dmaCleanup.Add(func() { mp.Release(t) }) + } + + case DmaMRRegDMABuf: + fdRelease, err := fd.prepareDMABufFD(t, attrs, &xlats) + if err != nil { + log.Warningf("rdmaproxy: REG_DMABUF_MR fd translation: %v", err) + return 0, err + } + defer fdRelease() + + case DmaCQCreate, DmaQPCreate: + mp, err := fd.prepareCreateDMA(t, staged) + if err != nil { + log.Warningf("rdmaproxy: CQ/QP CREATE page mirroring: %v", err) + return 0, err + } + if mp != nil { + cqqpMirror = mp + dmaCleanup.Add(func() { mp.Release(t) }) + } + + case DmaInvokeWrite: + // Legacy write-path REG_MR carries the guest MR address in its CORE_IN + // blob; mirror it just like the modern REG_MR method. Other write + // commands need no DMA handling and are forwarded opaquely. + mp, err := fd.prepareInvokeWriteRegMR(t, attrs, staged) + if err != nil { + log.Warningf("rdmaproxy: INVOKE_WRITE REG_MR page mirroring: %v", err) + return 0, err + } + if mp != nil { + mrMirror = mp + dmaCleanup.Add(func() { mp.Release(t) }) + } + } + + // Serialize the header and translated attrs back into one contiguous + // buffer, the layout the kernel expects, and forward it. RDMA_VERBS_IOCTL + // is _IOWR: the kernel writes outputs (new IDR handles, fds) back into the + // attr data fields, so the attrs are re-unmarshaled from buf after the call. + buf := make([]byte, hdr.Length) + ib.MarshalUnsafeUverbsAttrSlice(attrs, hdr.MarshalUnsafe(buf)) + n, errno := invokeUverbsIoctl(fd.hostFD, buf) + // Keep the staged buffers alive until the host syscall returns: their + // addresses were laundered through a uintptr into buf, so the GC cannot + // otherwise see that the in-flight ioctl still references them. + runtime.KeepAlive(staged) + ib.UnmarshalUnsafeUverbsAttrSlice(attrs, buf[ib.SizeofUverbsIoctlHdr:]) + + // asyncFDCleanup is armed if an async-event fd was installed below; it + // fires if the final copy-out fails (the guest never learned the number). + var asyncFDCleanup cleanup.Cleanup + defer asyncFDCleanup.Clean() + + if errno == 0 { + handleAttr := findAttr(attrs, schema.HandleAttr) + haveHandle := handleAttr != nil + var handle uint32 + if haveHandle { + // uverbs object handles are u32 (ib_uobject.id / __u32 mr_handle), + // zero-extended into the 8-byte data field. + handle = uint32(handleAttr.Data) + } + switch schema.Dma { + case DmaMRReg: + if mrMirror != nil { + // The object was created and the hardware now references the + // mirrored pages, so the mirror must outlive this call. Keep + // it whether or not the handle was found; if it wasn't, the + // mirror is untracked and unpins only at process teardown. + dmaCleanup.Release() + if haveHandle { + fd.pinned.addMR(handle, mrMirror) + } else { + log.Warningf("rdmaproxy: REG_MR succeeded but handle attr missing; leaking mirror") + } + } + case DmaMRRegDMABuf: + if haveHandle { + // No pages to mirror; track a sentinel so DEREG matches. + fd.pinned.addMR(handle, &MirroredPages{}) + } + case DmaMRDestroy: + if haveHandle { + if mp := fd.pinned.removeMR(handle); mp != nil { + mp.Release(t) + } + } + case DmaCQCreate, DmaQPCreate: + if cqqpMirror != nil { + dmaCleanup.Release() + if haveHandle { + fd.pinned.addDMABufs(handle, cqqpMirror) + } else { + log.Warningf("rdmaproxy: CQ/QP CREATE succeeded but handle attr missing; leaking mirror") + } + } + case DmaCQDestroy, DmaQPDestroy: + if haveHandle { + if bufs := fd.pinned.removeDMABufs(handle); bufs != nil { + bufs.Release(t) + } + } + case DmaAsyncAlloc: + undo, err := fd.wrapAsyncEventFD(t, handleAttr) + if err != nil { + log.Warningf("rdmaproxy: async event fd wrap: %v", err) + } else if undo != nil { + asyncFDCleanup.Add(undo) + } + case DmaInvokeWrite: + switch invokeWriteCmd(attrs) { + case ib.IB_USER_VERBS_CMD_REG_MR: + if mrMirror != nil { + // The MR was created and the hardware now references the + // mirrored pages, so keep the mirror regardless. + dmaCleanup.Release() + var resp ib.UverbsRegMRResp + if coreOut := staged[ib.UVERBS_ATTR_CORE_OUT]; coreOut != nil && len(coreOut) >= resp.SizeBytes() { + resp.UnmarshalBytes(coreOut) + fd.pinned.addMR(resp.MRHandle, mrMirror) + } else { + log.Warningf("rdmaproxy: write REG_MR succeeded but no CORE_OUT handle; leaking mirror") + } + } + case ib.IB_USER_VERBS_CMD_DEREG_MR: + // ib_uverbs_dereg_mr is {u32 mr_handle}: carried inline when + // CORE_IN is <=8 bytes, else the first u32 of the staged buffer. + // Read it either way so the mirror is released regardless of how + // the guest encoded CORE_IN. + if a := findAttr(attrs, ib.UVERBS_ATTR_CORE_IN); a != nil { + mrHandle, ok := uint32(a.Data), a.Len <= 8 + if a.Len > 8 { + if sb := staged[ib.UVERBS_ATTR_CORE_IN]; len(sb) >= 4 { + mrHandle, ok = hostarch.ByteOrder.Uint32(sb), true + } + } + if ok { + if mp := fd.pinned.removeMR(mrHandle); mp != nil { + mp.Release(t) + } + } + } + } + } + } + + // Copy PTR_OUT staging buffers back to their guest pointers and restore + // every rewritten data field so the guest observes only its own values. + for i := range xlats { + x := &xlats[i] + if errno == 0 && x.outBuf != nil { + // Best-effort: the guest pointer was validated on the way in. + t.CopyOutBytes(hostarch.Addr(x.orig), x.outBuf) + } + x.attr.Data = x.orig + } + // Only the attrs are copied back: the kernel never modifies the header. + if _, err := ib.CopyUverbsAttrSliceOut(t, attrsAddr, attrs); err != nil { + // The guest never received an installed async-event fd number. + return 0, err + } + asyncFDCleanup.Release() + + if errno != 0 { + return n, errno + } + return n, nil +} + +// copyInPtr stages a guest PTR_IN payload of len bytes into a fresh sentry +// buffer. length is the attribute's u16 wire length, so sb is at most 64 KiB. +func (fd *uverbsFD) copyInPtr(t *kernel.Task, guestPtr uint64, length uint16) ([]byte, error) { + sb := make([]byte, length) + if _, err := t.CopyInBytes(hostarch.Addr(guestPtr), sb); err != nil { + return nil, err + } + return sb, nil +} + +// translateInputFD resolves an app fd referencing a proxied async-event FD to +// its underlying host fd. The caller takes a ref on the returned file, which +// must be released once the host fd is no longer needed. +func (fd *uverbsFD) translateInputFD(t *kernel.Task, appFD int32) (int, *vfs.FileDescription, error) { + if appFD < 0 { + return 0, nil, linuxerr.EINVAL + } + file := t.GetFile(appFD) + if file == nil { + return 0, nil, linuxerr.EBADF + } + afd, ok := file.Impl().(*asyncEventFD) + if !ok { + log.Warningf("rdmaproxy: unsupported input fd=%d type %T (not a proxied RDMA fd)", appFD, file.Impl()) + file.DecRef(t) + return 0, nil, linuxerr.EINVAL + } + return int(afd.hostFD), file, nil +} + +// translateHostBackedFD resolves an app fd backed by a host fd (e.g. an +// nvproxy dma-buf export) to its host fd via vfs.HostFDProvider. The caller +// takes a ref on the returned file, which must be released once the host fd is +// no longer needed. +func (fd *uverbsFD) translateHostBackedFD(t *kernel.Task, appFD int32) (int, *vfs.FileDescription, error) { + if appFD < 0 { + return 0, nil, linuxerr.EINVAL + } + file := t.GetFile(appFD) + if file == nil { + return 0, nil, linuxerr.EBADF + } + hp, ok := file.Impl().(vfs.HostFDProvider) + if !ok { + log.Warningf("rdmaproxy: unsupported fd=%d type %T (not a vfs.HostFDProvider)", appFD, file.Impl()) + file.DecRef(t) + return 0, nil, linuxerr.EINVAL + } + return hp.HostFD(), file, nil +} + +// findAttr returns the attribute with the given id, or nil. +func findAttr(attrs []ib.UverbsAttr, id uint16) *ib.UverbsAttr { + for i := range attrs { + if attrs[i].AttrID == id { + return &attrs[i] + } + } + return nil +} + +// invokeWriteCmd returns the legacy write command number carried inline in the +// INVOKE_WRITE WRITE_CMD attribute, or math.MaxUint32 if absent. +func invokeWriteCmd(attrs []ib.UverbsAttr) uint32 { + a := findAttr(attrs, ib.UVERBS_ATTR_WRITE_CMD) + if a == nil { + return ^uint32(0) + } + return uint32(a.Data) +} + +// prepareInvokeWriteRegMR mirrors the guest MR pages for a legacy write-path +// REG_MR (the guest start/length live in the CORE_IN ib_uverbs_reg_mr blob, +// which the generic loop already copied into a sentry buffer). It rewrites the +// start field in that buffer to the sentry-side address. Returns nil for any +// non-REG_MR write command or a zero-length registration. +func (fd *uverbsFD) prepareInvokeWriteRegMR(t *kernel.Task, attrs []ib.UverbsAttr, staged map[uint16][]byte) (*MirroredPages, error) { + if invokeWriteCmd(attrs) != ib.IB_USER_VERBS_CMD_REG_MR { + return nil, nil + } + var cmd ib.UverbsRegMR + coreIn := staged[ib.UVERBS_ATTR_CORE_IN] + if coreIn == nil || len(coreIn) < cmd.SizeBytes() { + return nil, nil + } + cmd.UnmarshalBytes(coreIn) + if cmd.Length == 0 { + return nil, nil + } + mp, sentryVA, err := MirrorAppPages(t, cmd.Start, cmd.Length) + if err != nil { + return nil, err + } + cmd.Start = uint64(sentryVA) + cmd.MarshalBytes(coreIn) + return mp, nil +} + +// prepareMRReg mirrors the guest pages referenced by the REG_MR ADDR/LENGTH +// attributes (both inline u64 values) into the sentry and rewrites the inline +// ADDR to the sentry-side address. Returns nil if the request carried no +// address (e.g. an on-demand-paging or fd-backed registration). +func (fd *uverbsFD) prepareMRReg(t *kernel.Task, attrs []ib.UverbsAttr, xlats *[]xlat) (*MirroredPages, error) { + addr := findAttr(attrs, ib.UVERBS_ATTR_REG_MR_ADDR) + length := findAttr(attrs, ib.UVERBS_ATTR_REG_MR_LENGTH) + if addr == nil || addr.Len == 0 || length == nil || length.Len == 0 || length.Data == 0 { + return nil, nil + } + mp, sentryVA, err := MirrorAppPages(t, addr.Data, length.Data) + if err != nil { + return nil, err + } + *xlats = append(*xlats, xlat{attr: addr, orig: addr.Data}) + addr.Data = uint64(sentryVA) + return mp, nil +} + +// prepareDMABufFD translates the DMABUF fd carried inline in REG_DMABUF_MR's FD +// attribute (a PTR_IN u32) from an app fd to a host fd. +func (fd *uverbsFD) prepareDMABufFD(t *kernel.Task, attrs []ib.UverbsAttr, xlats *[]xlat) (func(), error) { + a := findAttr(attrs, ib.UVERBS_ATTR_REG_DMABUF_MR_FD) + if a == nil { + return nil, linuxerr.EINVAL + } + hostFD, file, err := fd.translateHostBackedFD(t, int32(a.Data)) + if err != nil { + return nil, err + } + *xlats = append(*xlats, xlat{attr: a, orig: a.Data}) + a.Data = uint64(uint32(hostFD)) + return func() { file.DecRef(t) }, nil +} + +// prepareCreateDMA hands the copied-in UHW_IN driver payload to the vendor +// driver so it can mirror the CQ/QP work-queue and doorbell buffers and rewrite +// the embedded addresses in place. +func (fd *uverbsFD) prepareCreateDMA(t *kernel.Task, staged map[uint16][]byte) (*PinnedDMABufs, error) { + uhw := staged[ib.UVERBS_ATTR_UHW_IN] + if uhw == nil { + // No driver payload (absent, or inline with no buffer pointers). + return nil, nil + } + return fd.driver.PrepareCreateDMA(t, uhw) +} + +// wrapAsyncEventFD wraps the host async-event fd the kernel wrote into the FD +// attribute a, installs a sentry FD, and rewrites a to the app fd number. +// Returns an undo closure to be run only if the subsequent copy-out to the +// guest fails. +func (fd *uverbsFD) wrapAsyncEventFD(t *kernel.Task, a *ib.UverbsAttr) (func(), error) { + if a == nil { + return nil, fmt.Errorf("ASYNC_EVENT_ALLOC response missing fd attr") + } + hostFD := int(int32(a.Data)) + if hostFD < 0 { + return nil, fmt.Errorf("kernel returned invalid async event fd %d", hostFD) + } + sentryFD, err := newAsyncEventFD(t, hostFD) // takes ownership of hostFD. + if err != nil { + return nil, err + } + a.Data = uint64(uint32(sentryFD)) + log.Infof("rdmaproxy: installed async event fd -> app fd %d", sentryFD) + return func() { + if f := t.FDTable().Remove(t, sentryFD); f != nil { + f.DecRef(t) + } + }, nil +} + +// Read implements vfs.FileDescriptionImpl.Read, forwarding async-event reads to +// the host fd. +func (fd *uverbsFD) Read(ctx context.Context, dst usermem.IOSequence, opts vfs.ReadOptions) (int64, error) { + return readProxiedEventFD(ctx, fd.hostFD, dst) +} + +// Write rejects direct writes: the legacy uverbs write() command interface is +// not implemented; modern rdma-core uses RDMA_VERBS_IOCTL exclusively. +func (fd *uverbsFD) Write(ctx context.Context, src usermem.IOSequence, opts vfs.WriteOptions) (int64, error) { + log.Warningf("rdmaproxy: unsupported legacy write(2) verbs interface") + return 0, linuxerr.EINVAL +} + +// Read implements vfs.FileDescriptionImpl.Read for asyncEventFD. +func (fd *asyncEventFD) Read(ctx context.Context, dst usermem.IOSequence, opts vfs.ReadOptions) (int64, error) { + return readProxiedEventFD(ctx, fd.hostFD, dst) +} + +// ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap. +func (fd *uverbsFD) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error { + if err := vfs.GenericProxyDeviceConfigureMMap(&fd.vfsfd, fd, opts); err != nil { + log.Warningf("rdmaproxy: mmap hostFD=%d: %v", fd.hostFD, err) + return err + } + return nil +} + +// Translate implements memmap.Mappable.Translate. +func (fd *uverbsFD) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { + return []memmap.Translation{ + { + Source: optional, + File: &fd.memmapFile, + Offset: optional.Start, + Perms: hostarch.AnyAccess, + }, + }, nil +} diff --git a/pkg/sentry/devices/rdmaproxy/ioctl_unsafe.go b/pkg/sentry/devices/rdmaproxy/ioctl_unsafe.go new file mode 100644 index 00000000000..beff34c7361 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/ioctl_unsafe.go @@ -0,0 +1,212 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file isolates the unsafe.Pointer and raw-syscall surface of the rdma +// proxy: forwarding the RDMA_VERBS_IOCTL to the host, reading from host FDs, +// and mirroring app pages into the sentry's address space via mmap/mremap. + +package rdmaproxy + +import ( + "runtime" + "unsafe" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/ib" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/atomicbitops" + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/fdnotifier" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/mm" + "gvisor.dev/gvisor/pkg/usermem" + "gvisor.dev/gvisor/pkg/waiter" +) + +// maxMirrorLen caps a single DMA-buffer mirror, bounding the memory a malicious +// guest can force the sentry to pin. 64 GiB is far above any legitimate RDMA +// work-queue or host bounce buffer while still refusing an obviously abusive +// length. +const maxMirrorLen = 64 << 30 + +// invokeUverbsIoctl forwards a pre-built RDMA_VERBS_IOCTL buffer to the host +// kernel. buf must already have its attribute data pointers rewritten to +// sentry-side addresses, and every referenced buffer must be kept alive across +// this call (see the runtime.KeepAlive in handleRDMAVerbsIoctl). +func invokeUverbsIoctl(hostFD int32, buf []byte) (uintptr, unix.Errno) { + n, _, errno := unix.Syscall(unix.SYS_IOCTL, + uintptr(hostFD), + uintptr(ib.RDMAVerbsIoctl), + uintptr(unsafe.Pointer(&buf[0]))) + runtime.KeepAlive(buf) + return n, errno +} + +// sentryDataPtr returns the address of the first byte of sb as a uint64 for +// placing into an ioctl attribute data field. +func sentryDataPtr(sb []byte) uint64 { + return uint64(uintptr(unsafe.Pointer(&sb[0]))) +} + +// readHostFDNonblocking issues a read(2) on hostFD into buf. The host FD is +// expected to be in O_NONBLOCK mode; callers translate EAGAIN/EWOULDBLOCK into +// ErrWouldBlock. +func readHostFDNonblocking(hostFD int32, buf []byte) (int, unix.Errno) { + if len(buf) == 0 { + return 0, 0 + } + n, _, errno := unix.RawSyscall(unix.SYS_READ, + uintptr(hostFD), + uintptr(unsafe.Pointer(&buf[0])), + uintptr(len(buf))) + return int(n), errno +} + +// eventReadBufLen bounds a single guest read(2) of a proxied event FD. Both the +// uverbs cdev async path and the dedicated async-event FD only ever deliver +// fixed 16-byte struct ib_uverbs_async_event_desc records (8-byte completion +// descriptors on a comp channel), so a small stack buffer drains several at +// once without a per-read heap allocation. +const eventReadBufLen = 128 + +// readProxiedEventFD services a guest read(2) on a proxied host event fd (the +// uverbs FD async stream, or an async-event FD). It is not the ioctl datapath: +// app blocks here waiting for async events that a healthy fabric never raises. +func readProxiedEventFD(ctx context.Context, hostFD int32, dst usermem.IOSequence) (int64, error) { + n := int(dst.NumBytes()) + if n == 0 { + return 0, nil + } + if fdnotifier.NonBlockingPoll(hostFD, waiter.ReadableEvents) == 0 { + return 0, linuxerr.ErrWouldBlock + } + if n > eventReadBufLen { + n = eventReadBufLen + } + var buf [eventReadBufLen]byte + got, errno := readHostFDNonblocking(hostFD, buf[:n]) + if errno != 0 { + if errno == unix.EAGAIN || errno == unix.EWOULDBLOCK { + return 0, linuxerr.ErrWouldBlock + } + return 0, errno + } + if got == 0 { + return 0, nil + } + w, err := dst.CopyOut(ctx, buf[:got]) + return int64(w), err +} + +var madvPopulateWriteDisabled atomicbitops.Bool + +// MirrorAppPages pins the app pages backing [addr, addr+length) and maps them +// contiguously into the sentry's address space so the host kernel can +// pin_user_pages on them for DMA. It returns the mirror and the sentry VA +// corresponding to addr. +func MirrorAppPages(t *kernel.Task, addr, length uint64) (*MirroredPages, uintptr, error) { + if length == 0 || length > maxMirrorLen { + return nil, 0, linuxerr.EINVAL + } + alignedStart := hostarch.Addr(addr).RoundDown() + alignedEnd, ok := hostarch.Addr(addr + length).RoundUp() + if !ok || uint64(alignedEnd) <= addr { + return nil, 0, linuxerr.EINVAL + } + alignedLen := uint64(alignedEnd - alignedStart) + + appAR, ok := alignedStart.ToRange(alignedLen) + if !ok { + return nil, 0, linuxerr.EINVAL + } + + at := hostarch.ReadWrite + prs, pinErr := t.MemoryManager().Pin(t, appAR, at, false /* ignorePermissions */) + if pinErr != nil { + return nil, 0, pinErr + } + + cu := cleanup.Make(func() { mm.Unpin(prs) }) + defer cu.Clean() + + // Prefer a single contiguous internal mapping if the pinned range is one + // physically contiguous run. + var m uintptr + mOwned := false + if len(prs) == 1 { + pr := prs[0] + ims, err := pr.File.MapInternal(memmap.FileRange{Start: pr.Offset, End: pr.Offset + uint64(pr.Source.Length())}, at) + if err != nil { + return nil, 0, err + } + if ims.NumBlocks() == 1 { + m = ims.Head().Addr() + } + } + + // Otherwise reserve a contiguous sentry range and remap each internal + // mapping into it. + if m == 0 { + var errno unix.Errno + m, _, errno = unix.RawSyscall6(unix.SYS_MMAP, 0, uintptr(alignedLen), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANONYMOUS, ^uintptr(0), 0) + if errno != 0 { + return nil, 0, errno + } + mOwned = true + cu.Add(func() { + unix.RawSyscall(unix.SYS_MUNMAP, m, uintptr(alignedLen), 0) + }) + sentryAddr := m + for _, pr := range prs { + ims, err := pr.File.MapInternal(memmap.FileRange{Start: pr.Offset, End: pr.Offset + uint64(pr.Source.Length())}, at) + if err != nil { + return nil, 0, err + } + for !ims.IsEmpty() { + im := ims.Head() + if _, _, errno := unix.RawSyscall6(unix.SYS_MREMAP, im.Addr(), 0, uintptr(im.Len()), linux.MREMAP_MAYMOVE|linux.MREMAP_FIXED, sentryAddr, 0); errno != 0 { + return nil, 0, errno + } + sentryAddr += uintptr(im.Len()) + ims = ims.Tail() + } + } + } + + // Best-effort pre-fault to avoid mmap_lock contention on first DMA. + // MADV_POPULATE_WRITE needs Linux 5.14; if it is unavailable the failure is + // permanent, so stop retrying it after the first failure. + if !madvPopulateWriteDisabled.Load() { + if _, _, errno := unix.Syscall(unix.SYS_MADVISE, m, uintptr(alignedLen), unix.MADV_POPULATE_WRITE); errno != 0 { + if !madvPopulateWriteDisabled.Swap(true) { + log.Infof("rdmaproxy: disabling MADV_POPULATE_WRITE pre-fault: %s", errno) + } + } + } + + mp := &MirroredPages{prs: prs} + if mOwned { + mp.m = m + mp.len = uintptr(alignedLen) + } + cu.Release() + + sentryVA := m + uintptr(addr-uint64(alignedStart)) + return mp, sentryVA, nil +} diff --git a/pkg/sentry/devices/rdmaproxy/rdmaproxy.go b/pkg/sentry/devices/rdmaproxy/rdmaproxy.go new file mode 100644 index 00000000000..9413183633f --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/rdmaproxy.go @@ -0,0 +1,411 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rdmaproxy implements a passthrough proxy for /dev/infiniband/uverbs* +// devices, enabling RDMA support inside gVisor sandboxes. +// +// # Design +// +// The proxy forwards the modern UVERBS ioctl interface (RDMA_VERBS_IOCTL = +// _IOWR(0x1b, 1, struct ib_uverbs_ioctl_hdr)) to a host uverbs FD, translating +// guest pointers, file descriptors and DMA buffers along the way. Every request +// is validated against an explicit per-(object, method) schema (see schema.go); +// an unmodeled object/method or an unexpected attribute is rejected rather than +// blindly forwarded. The schema — not any probing of guest memory — determines +// how each attribute's data field is interpreted, which is what keeps a +// malicious guest from coercing the proxy into an out-of-bounds copy. +// +// # Host ABI requirements +// +// The proxy targets the UVERBS ioctl object/method/attribute IDs that reached +// mainline in Linux 4.20 (Dec 2018) via include/uapi/rdma/ib_user_ioctl_cmds.h. +// Per upstream policy these IDs are additive only — old IDs are never +// repurposed — so newer kernels remain compatible; older kernels are not +// supported. +// +// At Register time the proxy checks the kernel's IB_USER_VERBS_ABI_VERSION +// (collected host-side from /sys/class/infiniband_verbs/abi_version, since the +// sentry runs chrooted and seccomp-confined and cannot read sysfs itself) and +// refuses to attach on a mismatch. That value has been frozen at 6 in upstream +// Linux for over a decade; the kernel bumps it only on a userspace-breaking +// change. +package rdmaproxy + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + "sync" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/ib" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/devutil" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/fdnotifier" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/fsutil" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/mm" + "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/waiter" +) + +// expectedUverbsABIVersion is the value of IB_USER_VERBS_ABI_VERSION the proxy +// is built against. Frozen at 6 in upstream Linux for over a decade. +const expectedUverbsABIVersion = 6 + +// uverbsClassDir is the sysfs class directory for the uverbs subsystem, where +// the host kernel publishes the class-wide abi_version. +const uverbsClassDir = "/sys/class/infiniband_verbs" + +// uverbsDevice implements vfs.Device for /dev/infiniband/uverbs*. +type uverbsDevice struct { + // devName is the device filename, e.g. "uverbs0". Distinct from the + // kernel minor number (e.g. 192) used for VFS registration. + devName string + // driver is the per-vendor plug-in attached to this device, looked up by + // name at Register time. Always non-nil: Register refuses to register a + // device whose host PCI driver has no plug-in. + driver Driver +} + +// Open implements vfs.Device.Open. +func (dev *uverbsDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { + devClient := devutil.GoferClientFromContext(ctx) + if devClient == nil { + log.Warningf("devutil.CtxDevGoferClient is not set") + return nil, linuxerr.ENOENT + } + devRelPath := filepath.Join("infiniband", dev.devName) + hostFD, err := devClient.OpenAt(ctx, devRelPath, opts.Flags) + if err != nil { + log.Warningf("rdmaproxy: open host device %s: %v", devRelPath, err) + return nil, err + } + fd := &uverbsFD{ + hostFD: int32(hostFD), + driver: dev.driver, + } + if err := fdnotifier.AddFD(fd.hostFD, &fd.queue); err != nil { + unix.Close(hostFD) + return nil, err + } + fd.memmapFile.SetFD(int(fd.hostFD)) + if err := fd.vfsfd.Init(fd, opts.Flags, auth.CredentialsFromContext(ctx), mnt, vfsd, &vfs.FileDescriptionOptions{ + UseDentryMetadata: true, + }); err != nil { + fdnotifier.RemoveFD(fd.hostFD) + unix.Close(hostFD) + return nil, err + } + return &fd.vfsfd, nil +} + +// MirroredPages tracks application pages pinned and mapped into the sentry's +// address space so the host kernel can pin_user_pages on them for DMA. +type MirroredPages struct { + prs []mm.PinnedRange + // If m != 0, it's a sentry-side mmap we own and must munmap on release. + m uintptr + len uintptr +} + +// Release tears down the sentry-side mapping and unpins the underlying app +// pages. +func (mp *MirroredPages) Release(ctx context.Context) { + if mp.m != 0 { + if _, _, errno := unix.RawSyscall(unix.SYS_MUNMAP, mp.m, mp.len, 0); errno != 0 { + log.Warningf("rdmaproxy: munmap %#x-%#x: %v", mp.m, mp.m+mp.len, errno) + } + } + mm.Unpin(mp.prs) +} + +// PinnedDMABufs tracks the buf + doorbell mirrors for a single CQ or QP. Driver +// plug-ins return this from PrepareCreateDMA; the core stores it against the +// resulting CQ/QP handle until DESTROY. +type PinnedDMABufs struct { + // Buf is the work-queue buffer mirror, or nil if the driver produced none. + Buf *MirroredPages + // DB is the doorbell page mirror, or nil if the driver produced none. + DB *MirroredPages +} + +// Release tears down both buffer mirrors. Safe to call with nil fields. +func (p *PinnedDMABufs) Release(ctx context.Context) { + if p.Buf != nil { + p.Buf.Release(ctx) + } + if p.DB != nil { + p.DB.Release(ctx) + } +} + +// pinnedResources tracks the app memory mirrored on behalf of one uverbs +// FD's MRs/CQs/QPs. Handles are scoped to the uverbs context (the underlying +// host FD), so this state lives per-uverbsFD. +type pinnedResources struct { + mu sync.Mutex + mrs map[uint32]*MirroredPages + // dmaBufs tracks CQ and QP mirrors together: object handles are unique + // within a uverbs context's IDR namespace, so CQ and QP ids never collide. + dmaBufs map[uint32]*PinnedDMABufs +} + +// addMR records the page mirror for a successful REG_MR. +func (p *pinnedResources) addMR(handle uint32, mp *MirroredPages) { + p.mu.Lock() + defer p.mu.Unlock() + if p.mrs == nil { + p.mrs = make(map[uint32]*MirroredPages) + } + p.mrs[handle] = mp +} + +// addDMABufs records the buf+db mirrors for a successful CQ/QP CREATE. +func (p *pinnedResources) addDMABufs(handle uint32, bufs *PinnedDMABufs) { + p.mu.Lock() + defer p.mu.Unlock() + if p.dmaBufs == nil { + p.dmaBufs = make(map[uint32]*PinnedDMABufs) + } + p.dmaBufs[handle] = bufs +} + +// removeMR removes and returns the mirror for handle, or nil if absent. The +// caller takes ownership of the returned mirror. +func (p *pinnedResources) removeMR(handle uint32) *MirroredPages { + p.mu.Lock() + defer p.mu.Unlock() + mp, ok := p.mrs[handle] + if !ok { + return nil + } + delete(p.mrs, handle) + return mp +} + +// removeDMABufs removes and returns the bufs for a CQ/QP handle, or nil if +// absent. The caller takes ownership of the returned bufs. +func (p *pinnedResources) removeDMABufs(handle uint32) *PinnedDMABufs { + p.mu.Lock() + defer p.mu.Unlock() + bufs, ok := p.dmaBufs[handle] + if !ok { + return nil + } + delete(p.dmaBufs, handle) + return bufs +} + +// releaseAll drains every tracked mirror and releases the underlying pages. +// Used as a safety net at FD close for handles the application neglected to +// DEREG/DESTROY. Subsequent calls are no-ops. +func (p *pinnedResources) releaseAll(ctx context.Context) { + p.mu.Lock() + mrs, dmaBufs := p.mrs, p.dmaBufs + p.mrs, p.dmaBufs = nil, nil + p.mu.Unlock() + for _, mp := range mrs { + mp.Release(ctx) + } + for _, bufs := range dmaBufs { + bufs.Release(ctx) + } +} + +// uverbsFD implements vfs.FileDescriptionImpl for an opened uverbs device. +// +// uverbsFD is not savable; RDMA state is not checkpoint/restored. +type uverbsFD struct { + vfsfd vfs.FileDescription + vfs.FileDescriptionDefaultImpl + vfs.DentryMetadataFileDescriptionImpl + vfs.NoLockFD + memmap.MappableNoTrackMappings + + hostFD int32 + queue waiter.Queue + memmapFile fsutil.MmapNoInternalFile + + // driver is the per-vendor plug-in inherited from the originating + // uverbsDevice at Open time. Always non-nil; see uverbsDevice.driver. + driver Driver + + // pinned tracks app memory mirrored on behalf of MRs, CQs, and QPs + // registered or created through this uverbs FD. + pinned pinnedResources +} + +// Release implements vfs.FileDescriptionImpl.Release. +func (fd *uverbsFD) Release(ctx context.Context) { + fd.pinned.releaseAll(ctx) + fdnotifier.RemoveFD(fd.hostFD) + unix.Close(int(fd.hostFD)) +} + +// EventRegister implements waiter.Waitable.EventRegister. +func (fd *uverbsFD) EventRegister(e *waiter.Entry) error { + fd.queue.EventRegister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + fd.queue.EventUnregister(e) + return err + } + return nil +} + +// EventUnregister implements waiter.Waitable.EventUnregister. +func (fd *uverbsFD) EventUnregister(e *waiter.Entry) { + fd.queue.EventUnregister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + panic(fmt.Sprint("UpdateFD:", err)) + } +} + +// Readiness implements waiter.Waitable.Readiness. +func (fd *uverbsFD) Readiness(mask waiter.EventMask) waiter.EventMask { + return fdnotifier.NonBlockingPoll(fd.hostFD, mask) +} + +// Register registers a proxied uverbs device with the VFS at the fixed uverbs +// char-device major (ib.IB_UVERBS_MAJOR) and the given minor. devName is the +// device filename (e.g. "uverbs0"). +// +// driverName is the host PCI driver name (typically the DRIVER= field of +// /sys/class/infiniband//device/uevent, e.g. "mlx5_core") used to look +// up the vendor plug-in via LookupDriver. If it is empty or names a driver we +// do not model, registration fails: the plug-in is required to safely mirror +// CQ/QP DMA buffers, so a device without one cannot be proxied. +// +// The host RoCE netdev backing each uverbs device is expected to be moved, +// fully configured, into the sandbox netns before the application connects +// QPs. Upstream ib_core would resolve RoCE addressing in whichever netns the +// netdev lives, but MLNX_OFED's ib_core requires the source GID's netdev to +// be in the ioctl caller's netns (rdma_check_gid_user_access; ENODEV at +// MODIFY_QP otherwise), and the sentry issues the ioctls from the sandbox +// netns. The sentry needs no setns or extra capabilities. +// +// verbsABIVersion is the host's IB_USER_VERBS_ABI_VERSION collected host-side +// before the sandbox started. An empty string means it could not be collected. +func Register(vfsObj *vfs.VirtualFilesystem, devName string, minor uint32, driverName, verbsABIVersion string) error { + // Validate the kernel uverbs ABI version. A mismatch means the kernel + // ships an unfamiliar UVERBS interface (refuse rather than risk + // misinterpreting attribute layouts). + if v, err := strconv.Atoi(strings.TrimSpace(verbsABIVersion)); err != nil { + return fmt.Errorf("rdmaproxy: no usable uverbs ABI version (collected %q from %s/abi_version); refusing to register — expected version %d", verbsABIVersion, uverbsClassDir, expectedUverbsABIVersion) + } else if v != expectedUverbsABIVersion { + return fmt.Errorf("rdmaproxy: kernel uverbs ABI version %d does not match expected %d (collected from %s/abi_version) — refusing to register %s", v, expectedUverbsABIVersion, uverbsClassDir, devName) + } + + driver := LookupDriver(driverName) + if driver == nil { + return fmt.Errorf("rdmaproxy: no vendor plug-in for driver %q — refusing to register %s (supported: %v)", driverName, devName, RegisteredDrivers()) + } + log.Infof("rdmaproxy: registering %s with major=%d minor=%d driver=%s (requested=%q)", devName, ib.IB_UVERBS_MAJOR, minor, driver.Name(), driverName) + return vfsObj.RegisterDevice(vfs.CharDevice, ib.IB_UVERBS_MAJOR, minor, &uverbsDevice{devName: devName, driver: driver}, &vfs.RegisterDeviceOptions{ + GroupName: "infiniband", + Pathname: filepath.Join("infiniband", devName), + FilePerms: 0666, + }) +} + +// asyncEventFD wraps a host FD for RDMA async event delivery. The kernel +// creates this FD via UVERBS_METHOD_ASYNC_EVENT_ALLOC; rdma-core reads async +// events from it via read(2). Input FD attributes referencing an async event +// (CQ/QP EVENT_FD) are translated back to this host FD at ioctl time by +// resolving the app FD through the task's FD table, which correctly handles +// FD-number recycling across application processes. +type asyncEventFD struct { + vfsfd vfs.FileDescription + vfs.FileDescriptionDefaultImpl + vfs.DentryMetadataFileDescriptionImpl + vfs.NoLockFD + + hostFD int32 + queue waiter.Queue +} + +// Release implements vfs.FileDescriptionImpl.Release. +func (fd *asyncEventFD) Release(ctx context.Context) { + fdnotifier.RemoveFD(fd.hostFD) + unix.Close(int(fd.hostFD)) +} + +// EventRegister implements waiter.Waitable.EventRegister. +func (fd *asyncEventFD) EventRegister(e *waiter.Entry) error { + fd.queue.EventRegister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + fd.queue.EventUnregister(e) + return err + } + return nil +} + +// EventUnregister implements waiter.Waitable.EventUnregister. +func (fd *asyncEventFD) EventUnregister(e *waiter.Entry) { + fd.queue.EventUnregister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + panic(fmt.Sprint("UpdateFD:", err)) + } +} + +// Readiness implements waiter.Waitable.Readiness. +func (fd *asyncEventFD) Readiness(mask waiter.EventMask) waiter.EventMask { + return fdnotifier.NonBlockingPoll(fd.hostFD, mask) +} + +// newAsyncEventFD wraps a host async-event FD in a sentry FileDescription and +// installs it in the task's FD table. Returns the app FD number. +// newAsyncEventFD takes ownership of hostFD. On success, hostFD ownership is +// transferred to the asyncEventFD. +func newAsyncEventFD(t *kernel.Task, hostFD int) (int32, error) { + vfsObj := t.Kernel().VFS() + vd := vfsObj.NewAnonVirtualDentry("[rdma-async-event]") + defer vd.DecRef(t) + + if err := unix.SetNonblock(hostFD, true); err != nil { + unix.Close(hostFD) + return -1, fmt.Errorf("SetNonblock: %w", err) + } + + afd := &asyncEventFD{ + hostFD: int32(hostFD), + } + if err := fdnotifier.AddFD(afd.hostFD, &afd.queue); err != nil { + unix.Close(hostFD) + return -1, err + } + if err := afd.vfsfd.Init(afd, linux.O_RDONLY, t.Credentials(), vd.Mount(), vd.Dentry(), &vfs.FileDescriptionOptions{ + UseDentryMetadata: true, + DenyPRead: true, + DenyPWrite: true, + }); err != nil { + fdnotifier.RemoveFD(afd.hostFD) + unix.Close(hostFD) + return -1, err + } + // From here hostFD ownership is transferred to afd. + defer afd.vfsfd.DecRef(t) + + appFD, err := t.NewFDFrom(0, &afd.vfsfd, kernel.FDFlags{CloseOnExec: true}) + if err != nil { + return -1, err + } + return appFD, nil +} diff --git a/pkg/sentry/devices/rdmaproxy/schema.go b/pkg/sentry/devices/rdmaproxy/schema.go new file mode 100644 index 00000000000..b1e1a36ae84 --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/schema.go @@ -0,0 +1,325 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdmaproxy + +import ( + "sync" + + "gvisor.dev/gvisor/pkg/abi/ib" +) + +// This file defines the explicit, allowlisting schema that drives translation +// of RDMA_VERBS_IOCTL requests. Every (object, method) pair the proxy accepts +// is described here, and every attribute within it carries a declared type. +// +// The declared type — NOT any runtime probing of the guest's memory — decides +// whether an attribute's data field is an inline value, a user pointer to copy +// in, an output buffer to allocate, an object handle to pass through, or a file +// descriptor to translate. This is the security-critical difference from a +// probe-by-CopyIn design: a guest cannot coerce the proxy into treating an +// output pointer as input (arbitrary write) or an inline value as a pointer +// (arbitrary read) by lying about its memory, because the direction is fixed by +// the kernel ABI, which is additive-only and stable since Linux 4.20. +// +// Object/method/attribute IDs are defined in pkg/abi/ib; the schemas below +// are verified against the method definitions in +// drivers/infiniband/core/uverbs_std_types*.c. + +// AttrType classifies how the proxy must translate an attribute's data field +// before forwarding the ioctl to the host and after it returns. +type AttrType uint8 + +const ( + // AttrPtrIn is an input value. If len<=8 the data field holds the value + // inline and is forwarded untouched; if len>8 the data field is a guest + // pointer whose contents are copied into a sentry buffer and the field is + // rewritten to the sentry address. + AttrPtrIn AttrType = iota + // AttrPtrOut is an output buffer. The data field is ALWAYS a guest + // pointer (even for 4-byte outputs); the proxy allocates a sentry buffer, + // rewrites the field, and copies the buffer back to the guest pointer + // after the call. + AttrPtrOut + // AttrInline is an inline scalar (value in the data field) that a + // method-specific handler reads by value. The generic loop leaves it in + // place but rejects a guest-supplied pointer (len>8), which the handler + // would otherwise misread as the value. + AttrInline + // AttrIdr is an object handle (len==0, data=handle). It is passed through + // unchanged: with a single host uverbs context per FD the guest and host + // share one handle namespace. For UVERBS_ACCESS_NEW attrs the kernel + // writes the freshly allocated handle into the data field of the + // forwarded buffer, which reaches the guest via copy-out. + AttrIdr + // AttrFdIn is an input file descriptor (len==0, data=fd) referencing an + // object the proxy previously wrapped (an async-event FD). The guest fd + // number is translated to the host fd before the call and restored after. + AttrFdIn + // AttrFdNew is an output file descriptor (len==0). The kernel installs a + // host fd in the sentry process and writes its number into the data + // field; the proxy wraps it in a sentry FD and rewrites the field. + AttrFdNew + // AttrRawFd is a raw file descriptor (len==0, data_s64=fd), translated + // like AttrFdIn when non-negative. + AttrRawFd + // AttrUnsupported is a modeled-but-rejected attribute (e.g. UMEM or a + // buffer-FD path the proxy does not implement). Its presence in a request + // fails the ioctl with EINVAL rather than being silently forwarded. + AttrUnsupported +) + +// DmaKind selects the method-level DMA-mirroring / resource-tracking behavior +// applied in addition to the generic per-attribute translation. +type DmaKind uint8 + +const ( + DmaNone DmaKind = iota + // DmaMRReg mirrors the guest pages referenced by the REG_MR ADDR/LENGTH + // attributes and tracks the resulting MR handle for teardown. + DmaMRReg + // DmaMRRegDMABuf tracks the DMABUF MR handle (no page mirroring; the DMA + // buffer is resolved through the dma-buf framework, not guest VMAs). + DmaMRRegDMABuf + // DmaMRDestroy releases the pages mirrored for the MR being destroyed. + DmaMRDestroy + // DmaCQCreate / DmaQPCreate mirror the vendor DMA buffers via the driver + // plug-in and track the CQ/QP handle. + DmaCQCreate + DmaQPCreate + // DmaCQDestroy / DmaQPDestroy release the mirrors tracked at create. + DmaCQDestroy + DmaQPDestroy + // DmaAsyncAlloc wraps the newly allocated async-event host fd. + DmaAsyncAlloc + // DmaInvokeWrite handles the legacy write ABI wrapped in INVOKE_WRITE: it + // forwards CORE_IN/CORE_OUT opaquely, and for the write-path REG_MR/DEREG_MR + // commands mirrors the guest MR pages and tracks the MR handle. + DmaInvokeWrite +) + +// MethodSchema is the complete, verified description of one (object, method). +type MethodSchema struct { + Dma DmaKind + // HandleAttr is the attribute carrying this object's IDR handle, used for + // create/destroy resource tracking. Only meaningful when Dma tracks a + // handle. + HandleAttr uint16 + // Attrs maps attribute id -> type. Any attribute not present here causes + // the request to be rejected. + Attrs map[uint16]AttrType +} + +// schemas is the allowlist of supported (object, method) pairs, keyed by +// (object<<16 | method). Requests outside this set are rejected with EINVAL. +// Built on first use so that a runsc without RDMA never allocates it. +var ( + schemasOnce sync.Once + schemas map[uint32]*MethodSchema +) + +func SchemaKey(object, method uint16) uint32 { + return uint32(object)<<16 | uint32(method) +} + +func lookupSchema(object, method uint16) *MethodSchema { + schemasOnce.Do(func() { schemas = buildSchemas() }) + return schemas[SchemaKey(object, method)] +} + +func buildSchemas() map[uint32]*MethodSchema { + m := map[uint32]*MethodSchema{ + SchemaKey(ib.UVERBS_OBJECT_DEVICE, ib.UVERBS_METHOD_INVOKE_WRITE): { + // The compatibility wrapper for the legacy write ABI. CORE_IN / + // CORE_OUT are opaque command/response blobs (the kernel validates + // their contents); the DMA-relevant write commands are handled by + // DmaInvokeWrite. This one entry covers alloc_pd, dealloc_pd, + // modify_qp, query_qp, query_device, create_ah, etc. + Dma: DmaInvokeWrite, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_CORE_IN: AttrPtrIn, + ib.UVERBS_ATTR_CORE_OUT: AttrPtrOut, + ib.UVERBS_ATTR_WRITE_CMD: AttrInline, + ib.UVERBS_ATTR_UHW_IN: AttrPtrIn, + ib.UVERBS_ATTR_UHW_OUT: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_DEVICE, ib.UVERBS_METHOD_GET_CONTEXT): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS: AttrPtrOut, + ib.UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT: AttrPtrOut, + // FD_ARR passes an array of fds that would need per-element + // app->host translation; unused by mlx5 get_context, so + // reject rather than forward untranslated fds. + ib.UVERBS_ATTR_GET_CONTEXT_FD_ARR: AttrUnsupported, + ib.UVERBS_ATTR_UHW_IN: AttrPtrIn, + ib.UVERBS_ATTR_UHW_OUT: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_DEVICE, ib.UVERBS_METHOD_QUERY_CONTEXT): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS: AttrPtrOut, + ib.UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_DEVICE, ib.UVERBS_METHOD_QUERY_PORT): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_QUERY_PORT_PORT_NUM: AttrPtrIn, + ib.UVERBS_ATTR_QUERY_PORT_RESP: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_DEVICE, ib.UVERBS_METHOD_QUERY_GID_TABLE): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE: AttrPtrIn, + ib.UVERBS_ATTR_QUERY_GID_TABLE_FLAGS: AttrPtrIn, + ib.UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES: AttrPtrOut, + ib.UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_DEVICE, ib.UVERBS_METHOD_QUERY_GID_ENTRY): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_QUERY_GID_ENTRY_PORT: AttrPtrIn, + ib.UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX: AttrPtrIn, + ib.UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS: AttrPtrIn, + ib.UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_PD, ib.UVERBS_METHOD_PD_DESTROY): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_DESTROY_PD_HANDLE: AttrIdr, + }, + }, + // AH creation rides the legacy write path (DmaInvokeWrite); only + // destroy has a modern object method. + SchemaKey(ib.UVERBS_OBJECT_AH, ib.UVERBS_METHOD_AH_DESTROY): { + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_DESTROY_AH_HANDLE: AttrIdr, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_MR, ib.UVERBS_METHOD_REG_MR): { + Dma: DmaMRReg, HandleAttr: ib.UVERBS_ATTR_REG_MR_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_REG_MR_HANDLE: AttrIdr, + ib.UVERBS_ATTR_REG_MR_PD_HANDLE: AttrIdr, + ib.UVERBS_ATTR_REG_MR_DMA_HANDLE: AttrIdr, + ib.UVERBS_ATTR_REG_MR_IOVA: AttrPtrIn, + ib.UVERBS_ATTR_REG_MR_ADDR: AttrInline, + ib.UVERBS_ATTR_REG_MR_LENGTH: AttrInline, + ib.UVERBS_ATTR_REG_MR_ACCESS_FLAGS: AttrPtrIn, + ib.UVERBS_ATTR_REG_MR_FD: AttrRawFd, + ib.UVERBS_ATTR_REG_MR_FD_OFFSET: AttrPtrIn, + ib.UVERBS_ATTR_REG_MR_RESP_LKEY: AttrPtrOut, + ib.UVERBS_ATTR_REG_MR_RESP_RKEY: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_MR, ib.UVERBS_METHOD_REG_DMABUF_MR): { + Dma: DmaMRRegDMABuf, HandleAttr: ib.UVERBS_ATTR_REG_DMABUF_MR_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_REG_DMABUF_MR_HANDLE: AttrIdr, + ib.UVERBS_ATTR_REG_DMABUF_MR_PD_HANDLE: AttrIdr, + ib.UVERBS_ATTR_REG_DMABUF_MR_OFFSET: AttrPtrIn, + ib.UVERBS_ATTR_REG_DMABUF_MR_LENGTH: AttrPtrIn, + ib.UVERBS_ATTR_REG_DMABUF_MR_IOVA: AttrPtrIn, + // The DMABUF fd is translated from an app fd to a host fd in + // the DMABUF-specific pre-processing step. + ib.UVERBS_ATTR_REG_DMABUF_MR_FD: AttrInline, + ib.UVERBS_ATTR_REG_DMABUF_MR_ACCESS_FLAGS: AttrPtrIn, + ib.UVERBS_ATTR_REG_DMABUF_MR_RESP_LKEY: AttrPtrOut, + ib.UVERBS_ATTR_REG_DMABUF_MR_RESP_RKEY: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_MR, ib.UVERBS_METHOD_MR_DESTROY): { + Dma: DmaMRDestroy, HandleAttr: ib.UVERBS_ATTR_DESTROY_MR_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_DESTROY_MR_HANDLE: AttrIdr, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_CQ, ib.UVERBS_METHOD_CQ_CREATE): { + Dma: DmaCQCreate, HandleAttr: ib.UVERBS_ATTR_CREATE_CQ_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_CREATE_CQ_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_CQ_CQE: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_CQ_USER_HANDLE: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL: AttrFdIn, + ib.UVERBS_ATTR_CREATE_CQ_COMP_VECTOR: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_CQ_FLAGS: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_CQ_RESP_CQE: AttrPtrOut, + ib.UVERBS_ATTR_CREATE_CQ_EVENT_FD: AttrFdIn, + // The BUFFER_* / UMEM registration path is an alternative + // to the vendor UHW DMA buffers; mlx5 uses UHW, so these + // are rejected rather than half-supported. + ib.UVERBS_ATTR_CREATE_CQ_BUFFER_VA: AttrUnsupported, + ib.UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH: AttrUnsupported, + ib.UVERBS_ATTR_CREATE_CQ_BUFFER_FD: AttrUnsupported, + ib.UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET: AttrUnsupported, + ib.UVERBS_ATTR_CREATE_CQ_BUF_UMEM: AttrUnsupported, + ib.UVERBS_ATTR_UHW_IN: AttrPtrIn, + ib.UVERBS_ATTR_UHW_OUT: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_CQ, ib.UVERBS_METHOD_CQ_DESTROY): { + Dma: DmaCQDestroy, HandleAttr: ib.UVERBS_ATTR_DESTROY_CQ_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_DESTROY_CQ_HANDLE: AttrIdr, + ib.UVERBS_ATTR_DESTROY_CQ_RESP: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_QP, ib.UVERBS_METHOD_QP_CREATE): { + Dma: DmaQPCreate, HandleAttr: ib.UVERBS_ATTR_CREATE_QP_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_CREATE_QP_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_XRCD_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_PD_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_SRQ_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE: AttrIdr, + ib.UVERBS_ATTR_CREATE_QP_USER_HANDLE: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_QP_CAP: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_QP_TYPE: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_QP_FLAGS: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_QP_SOURCE_QPN: AttrPtrIn, + ib.UVERBS_ATTR_CREATE_QP_EVENT_FD: AttrFdIn, + ib.UVERBS_ATTR_CREATE_QP_RESP_CAP: AttrPtrOut, + ib.UVERBS_ATTR_CREATE_QP_RESP_QP_NUM: AttrPtrOut, + ib.UVERBS_ATTR_CREATE_QP_BUF_UMEM: AttrUnsupported, + ib.UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM: AttrUnsupported, + ib.UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM: AttrUnsupported, + ib.UVERBS_ATTR_UHW_IN: AttrPtrIn, + ib.UVERBS_ATTR_UHW_OUT: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_QP, ib.UVERBS_METHOD_QP_DESTROY): { + Dma: DmaQPDestroy, HandleAttr: ib.UVERBS_ATTR_DESTROY_QP_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_DESTROY_QP_HANDLE: AttrIdr, + ib.UVERBS_ATTR_DESTROY_QP_RESP: AttrPtrOut, + }, + }, + SchemaKey(ib.UVERBS_OBJECT_ASYNC_EVENT, ib.UVERBS_METHOD_ASYNC_EVENT_ALLOC): { + Dma: DmaAsyncAlloc, HandleAttr: ib.UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE, + Attrs: map[uint16]AttrType{ + ib.UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE: AttrFdNew, + }, + }, + } + // Merge the driver-namespace (object, method) pairs each registered vendor + // plug-in models (e.g. the mlx5 UAR); these carry no core DMA semantics. + rangeDrivers(func(d Driver) { + for k, s := range d.Schemas() { + m[k] = s + } + }) + return m +} diff --git a/pkg/sentry/devices/rdmaproxy/seccomp_filter.go b/pkg/sentry/devices/rdmaproxy/seccomp_filter.go new file mode 100644 index 00000000000..c2e1848716f --- /dev/null +++ b/pkg/sentry/devices/rdmaproxy/seccomp_filter.go @@ -0,0 +1,54 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdmaproxy + +import ( + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/ib" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/seccomp" +) + +// Filters returns the seccomp-bpf filters the RDMA proxy adds to the base +// sandbox filter. +func Filters() seccomp.SyscallRules { + return seccomp.MakeSyscallRules(map[uintptr]seccomp.SyscallRule{ + unix.SYS_IOCTL: seccomp.PerArg{ + seccomp.NonNegativeFD{}, + seccomp.EqualTo(uintptr(ib.RDMAVerbsIoctl)), + }, + unix.SYS_MREMAP: seccomp.PerArg{ + seccomp.AnyValue{}, + seccomp.EqualTo(0), // old_size + seccomp.AnyValue{}, + seccomp.EqualTo(linux.MREMAP_MAYMOVE | linux.MREMAP_FIXED), + seccomp.AnyValue{}, + seccomp.EqualTo(0), + }, + // The RDMA sysfs surface serves per-port dynamic state (the RoCE GID + // table, link state, counters) by reading host sysfs at access time via + // openat(-1, , O_RDONLY|O_NOFOLLOW) (see + // sys.hostFile.Generate). Without this rule a guest reading e.g. + // /sys/class/infiniband/*/ports/*/gids/* would SIGSYS-kill the sentry. + // dirfd is -1 (paths are absolute) and O_CREAT is forbidden, matching + // tpuproxy. + unix.SYS_OPENAT: seccomp.PerArg{ + seccomp.EqualTo(^uintptr(0)), + seccomp.AnyValue{}, + seccomp.MaskedEqual(unix.O_CREAT|unix.O_NOFOLLOW, unix.O_NOFOLLOW), + seccomp.AnyValue{}, + }, + }) +} diff --git a/pkg/sentry/fsimpl/proc/tasks_files.go b/pkg/sentry/fsimpl/proc/tasks_files.go index 6fe79fcfcac..08528155ec2 100644 --- a/pkg/sentry/fsimpl/proc/tasks_files.go +++ b/pkg/sentry/fsimpl/proc/tasks_files.go @@ -27,6 +27,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/kernel/version" "gvisor.dev/gvisor/pkg/sentry/ktime" "gvisor.dev/gvisor/pkg/sentry/usage" "gvisor.dev/gvisor/pkg/sentry/vfs" @@ -394,8 +395,7 @@ func (*versionData) Generate(ctx context.Context, buf *bytes.Buffer) error { // FIXME(mpratt): Using Version from the init task SyscallTable // disregards the different version a task may have (e.g., in a uts // namespace). - ver := kernelVersion(ctx) - fmt.Fprintf(buf, "%s version %s %s\n", ver.Sysname, ver.Release, ver.Version) + fmt.Fprintf(buf, "%s version %s %s\n", version.LinuxSysname, version.LinuxRelease(), version.LinuxVersion) return nil } @@ -442,23 +442,10 @@ var _ dynamicInode = (*cmdLineData)(nil) // Generate implements vfs.DynamicByteSource.Generate. func (*cmdLineData) Generate(ctx context.Context, buf *bytes.Buffer) error { - fmt.Fprintf(buf, "BOOT_IMAGE=/vmlinuz-%s-gvisor quiet\n", kernelVersion(ctx).Release) + fmt.Fprintf(buf, "BOOT_IMAGE=/vmlinuz-%s quiet\n", version.LinuxRelease()) return nil } -// kernelVersion returns the kernel version. -func kernelVersion(ctx context.Context) kernel.Version { - k := kernel.KernelFromContext(ctx) - init := k.GlobalInit() - if init == nil { - // Attempted to read before the init Task is created. This can - // only occur during startup, which should never need to read - // this file. - panic("Attempted to read version before initial Task is available") - } - return init.Leader().SyscallTable().Version -} - // devicesData backs /proc/devices. // // +stateify savable diff --git a/pkg/sentry/fsimpl/proc/tasks_sys.go b/pkg/sentry/fsimpl/proc/tasks_sys.go index 91f500f28f0..0f6b97b8cae 100644 --- a/pkg/sentry/fsimpl/proc/tasks_sys.go +++ b/pkg/sentry/fsimpl/proc/tasks_sys.go @@ -74,7 +74,7 @@ func (fs *filesystem) newSysDir(ctx context.Context, root *auth.Credentials, k * "keys": fs.newStaticDir(ctx, root, map[string]kernfs.Inode{ "maxkeys": fs.newMaxKeySizeFile(ctx, k, root), }), - "osrelease": fs.newInode(ctx, root, 0444, newStaticFile(version.LinuxRelease)), + "osrelease": fs.newInode(ctx, root, 0444, newStaticFile(version.LinuxRelease())), "ostype": fs.newInode(ctx, root, 0444, newStaticFile(version.LinuxSysname)), "version": fs.newInode(ctx, root, 0444, newStaticFile(version.LinuxVersion)), }), diff --git a/pkg/sentry/fsimpl/sys/rdma.go b/pkg/sentry/fsimpl/sys/rdma.go index 1310a9c16fc..3c705352fe8 100644 --- a/pkg/sentry/fsimpl/sys/rdma.go +++ b/pkg/sentry/fsimpl/sys/rdma.go @@ -58,6 +58,9 @@ type rdmaSysfsDirs struct { class map[string]kernfs.Inode // busPCIDevices contains the /sys/bus/pci/devices symlinks. busPCIDevices map[string]kernfs.Inode + // busPCIDrivers maps a kernel driver name to its + // /sys/bus/pci/drivers/ directory of bound-device back-symlinks. + busPCIDrivers map[string]kernfs.Inode // node is the /sys/devices/system/node subtree, or nil. node kernfs.Inode } @@ -105,6 +108,26 @@ func (fs *filesystem) newRDMASysfs(ctx context.Context, creds *auth.Credentials, classNet := map[string]string{} // netdev -> symlink target classPCIBus := map[string]string{} // bus ("0000:0c") -> symlink target + // driverByLeaf maps a leaf PCI function path to its kernel driver name + // (from the DRIVER= line of the leaf's uevent). Used to synthesize the + // device/driver symlink and /sys/bus/pci/drivers tree that libfabric's + // EFA provider resolves during device discovery. + driverByLeaf := map[string]string{} + for i := range snap.Devices { + leaf := snap.Devices[i].LeafPCI + for _, n := range snap.PCINodes { + if n.Path != leaf { + continue + } + for _, line := range strings.Split(n.Attrs["uevent"], "\n") { + if drv, ok := strings.CutPrefix(line, "DRIVER="); ok && rdma.SafeName(drv) { + driverByLeaf[leaf] = drv + } + } + } + } + classPCIDrivers := map[string][]string{} // driver -> leaf PCI paths bound to it + // 1. The canonical PCI hierarchy with per-level static attributes, plus // the "subsystem" symlink every PCI device carries. NCCL and other // consumers classify a directory as a PCI device by following @@ -135,12 +158,21 @@ func (fs *filesystem) newRDMASysfs(ctx context.Context, creds *auth.Credentials, if _, ok := d.files["local_cpulist"]; ok { d.files["local_cpulist"] = cpuListString(cores) } + // Raw PCI config space (binary). hwloc reads it to rebuild the PCI + // bridge hierarchy; without it aws-ofi-nccl's NCCL topology write fails. + if n.Config != nil { + d.files["config"] = string(n.Config) + } // Root complexes (pciXXXX:YY) carry no subsystem link and sit on // no parent bus; only function directories (BDFs) do. if rdma.IsBDF(path.Base(n.Path)) { // depth of n.Path below /sys == number of "../" to reach /sys. depth := strings.Count(n.Path, "/") + 1 d.symlinks["subsystem"] = strings.Repeat("../", depth) + "bus/pci" + if drv, ok := driverByLeaf[n.Path]; ok { + d.symlinks["driver"] = strings.Repeat("../", depth) + "bus/pci/drivers/" + drv + classPCIDrivers[drv] = append(classPCIDrivers[drv], n.Path) + } fs.addPCIBus(root, n.Path, classPCIBus) } } @@ -211,6 +243,7 @@ func (fs *filesystem) newRDMASysfs(ctx context.Context, creds *auth.Credentials, devices: map[string]kernfs.Inode{}, class: map[string]kernfs.Inode{}, busPCIDevices: map[string]kernfs.Inode{}, + busPCIDrivers: map[string]kernfs.Inode{}, } devicesTree, ok := root.children["devices"] if !ok { @@ -240,6 +273,17 @@ func (fs *filesystem) newRDMASysfs(ctx context.Context, creds *auth.Credentials, } } + // /sys/bus/pci/drivers// back-symlinks (the inverse of the + // device/driver links added above). libfabric's EFA provider realpath's + // the driver dir to confirm the bound driver during discovery. + for drv, leaves := range classPCIDrivers { + entries := map[string]kernfs.Inode{} + for _, leaf := range leaves { + entries[path.Base(leaf)] = kernfs.NewStaticSymlink(ctx, creds, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), "../../../../"+leaf) + } + out.busPCIDrivers[drv] = fs.newDir(ctx, creds, defaultSysDirMode, entries) + } + if snap.NUMA != nil { out.node = fs.buildNUMA(ctx, creds, snap.NUMA, cores) } diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index 03b358ef05e..f0438680811 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -137,6 +137,7 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt productName := "" busSub := make(map[string]kernfs.Inode) // /sys/bus pciDevices := make(map[string]kernfs.Inode) // /sys/bus/pci/devices + pciDrivers := make(map[string]kernfs.Inode) // /sys/bus/pci/drivers kernelSub := kernelDir(ctx, fs, creds) // /sys/kernel if opts.InternalData != nil { idata := opts.InternalData.(*InternalData) @@ -204,15 +205,23 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt for name, sub := range rdmaDirs.busPCIDevices { pciDevices[name] = sub } + for name, sub := range rdmaDirs.busPCIDrivers { + pciDrivers[name] = sub + } if rdmaDirs.node != nil { systemSub["node"] = rdmaDirs.node } } } - if len(pciDevices) > 0 { - busSub["pci"] = fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ - "devices": fs.newDir(ctx, creds, defaultSysDirMode, pciDevices), - }) + if len(pciDevices) > 0 || len(pciDrivers) > 0 { + pciSub := map[string]kernfs.Inode{} + if len(pciDevices) > 0 { + pciSub["devices"] = fs.newDir(ctx, creds, defaultSysDirMode, pciDevices) + } + if len(pciDrivers) > 0 { + pciSub["drivers"] = fs.newDir(ctx, creds, defaultSysDirMode, pciDrivers) + } + busSub["pci"] = fs.newDir(ctx, creds, defaultSysDirMode, pciSub) } devicesSub["system"] = fs.newDir(ctx, creds, defaultSysDirMode, systemSub) diff --git a/pkg/sentry/fsimpl/sys/sys_integration_test.go b/pkg/sentry/fsimpl/sys/sys_integration_test.go index b286a4c5564..691c1414d22 100644 --- a/pkg/sentry/fsimpl/sys/sys_integration_test.go +++ b/pkg/sentry/fsimpl/sys/sys_integration_test.go @@ -323,6 +323,10 @@ func TestEnableTPUProxyPathsV5(t *testing.T) { } } +// rdmaTestPCIConfig is a fake raw PCI config space blob; deliberately not +// valid UTF-8 to check binary-faithful mirroring. +var rdmaTestPCIConfig = []byte{0x86, 0x80, 0x0d, 0x02, 0x00, 0x00, 0x10, 0x00} + // newRDMATestSnapshot returns a snapshot with a ConnectX NIC behind a bridge // on domain 0000 and a GPU on an extended (VMD-style, 5-hex-digit) domain. func newRDMATestSnapshot() *rdma.Snapshot { @@ -331,10 +335,10 @@ func newRDMATestSnapshot() *rdma.Snapshot { VerbsABIVersion: "6\n", PCINodes: []rdma.PCINode{ {Path: "devices/pci0000:07"}, - {Path: "devices/pci0000:07/0000:07:01.0", Attrs: map[string]string{"class": "0x060400\n"}}, + {Path: "devices/pci0000:07/0000:07:01.0", Attrs: map[string]string{"class": "0x060400\n"}, Config: rdmaTestPCIConfig}, {Path: nicLeaf, Attrs: map[string]string{ "vendor": "0x15b3\n", - "uevent": "PCI_SLOT_NAME=0000:0c:00.0\n", + "uevent": "DRIVER=mlx5_core\nPCI_SLOT_NAME=0000:0c:00.0\n", // Host-view CPU affinity: must be rewritten to the sandbox's // single-node view. "numa_node": "1\n", @@ -399,12 +403,14 @@ func TestRDMASysfs(t *testing.T) { "pci_bus": linux.DT_DIR, "subsystem": linux.DT_LNK, "class": linux.DT_REG, + "config": linux.DT_REG, }, nicLeaf: { "infiniband": linux.DT_DIR, "infiniband_verbs": linux.DT_DIR, "net": linux.DT_DIR, "subsystem": linux.DT_LNK, + "driver": linux.DT_LNK, "vendor": linux.DT_REG, "uevent": linux.DT_REG, "numa_node": linux.DT_REG, @@ -455,6 +461,8 @@ func TestRDMASysfs(t *testing.T) { "0000:0c:00.0": linux.DT_LNK, "10000:e0:06.0": linux.DT_LNK, }, + "/bus/pci/drivers": {"mlx5_core": linux.DT_DIR}, + "/bus/pci/drivers/mlx5_core": {"0000:0c:00.0": linux.DT_LNK}, // The two host nodes collapse into a single sandbox node. "/devices/system/node": {"online": linux.DT_REG, "possible": linux.DT_REG, "node0": linux.DT_DIR}, "/devices/system/node/node0": { @@ -486,6 +494,13 @@ func TestRDMASysfs(t *testing.T) { nicLeaf + "/numa_node": "0\n", nicLeaf + "/local_cpulist": wantCPUList, "/devices/pci10000:e0/10000:e0:06.0/numa_node": "-1\n", + // Raw PCI config space is mirrored byte-faithfully. + "/devices/pci0000:07/0000:07:01.0/config": string(rdmaTestPCIConfig), + // The device/driver symlink and the /sys/bus/pci/drivers back-symlink + // resolve to the same PCI function (libfabric realpath's the former + // during EFA discovery). + nicLeaf + "/driver/0000:0c:00.0/vendor": "0x15b3\n", + "/bus/pci/drivers/mlx5_core/0000:0c:00.0/vendor": "0x15b3\n", } { got, err := readTestFile(s, p) if err != nil { diff --git a/pkg/sentry/kernel/BUILD b/pkg/sentry/kernel/BUILD index c0130a1c9ea..37147d372be 100644 --- a/pkg/sentry/kernel/BUILD +++ b/pkg/sentry/kernel/BUILD @@ -341,7 +341,6 @@ go_library( "user_counters_mutex.go", "uts_namespace.go", "vdso.go", - "version.go", ], imports = [ "gvisor.dev/gvisor/pkg/bpf", diff --git a/pkg/sentry/kernel/syscalls.go b/pkg/sentry/kernel/syscalls.go index ea80ddc49cd..b7840ca6e67 100644 --- a/pkg/sentry/kernel/syscalls.go +++ b/pkg/sentry/kernel/syscalls.go @@ -285,9 +285,6 @@ type SyscallTable struct { // Arch is the architecture that this syscall table targets. Arch arch.Arch - // The OS version that this syscall table implements. - Version Version - // AuditNumber is a numeric constant that represents the syscall table. If // non-zero, auditNumber must be one of the AUDIT_ARCH_* values defined by // linux/audit.h. diff --git a/pkg/sentry/kernel/version.go b/pkg/sentry/kernel/version.go deleted file mode 100644 index 678ef9e2725..00000000000 --- a/pkg/sentry/kernel/version.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kernel - -// Version defines the application-visible system version. -type Version struct { - // Operating system name (e.g. "Linux"). - Sysname string - - // Operating system release (e.g. "4.4-amd64"). - Release string - - // Operating system version. On Linux this takes the shape - // "#VERSION CONFIG_FLAGS TIMESTAMP" - // where: - // - VERSION is a sequence counter incremented on every successful build - // - CONFIG_FLAGS is a space-separated list of major enabled kernel features - // (e.g. "SMP" and "PREEMPT") - // - TIMESTAMP is the build timestamp as returned by `date` - Version string -} diff --git a/pkg/sentry/kernel/version/version.go b/pkg/sentry/kernel/version/version.go index 1052811a28b..94e40341b44 100644 --- a/pkg/sentry/kernel/version/version.go +++ b/pkg/sentry/kernel/version/version.go @@ -19,17 +19,41 @@ const ( // LinuxSysname is the OS name advertised by gVisor. LinuxSysname = "Linux" - // LinuxRelease is the Linux release version number advertised by gVisor. + // defaultRelease is the default Linux release version number advertised + // by gVisor. // // Must be high enough to satisfy the NT_GNU_ABI_TAG minimum-kernel check // performed by glibc's dynamic linker; otherwise dlopen() rejects modern // shared libraries (e.g. libQt6Core.so.6 requires >= 4.11.0) with a // misleading ENOENT. 4.19 is the final LTS of the Linux 4.x series, - // which keeps us on a 4.x base consistent with the syscall table ABI - // while providing headroom for typical modern userspace. The "-gvisor" - // suffix follows the distro-kernel convention (e.g. "-generic", "-azure"). - LinuxRelease = "4.19.0-gvisor" + // which keeps us on a 4.x base. The "-gvisor" suffix follows the + // distro-kernel convention (e.g. "-generic", "-azure"). + defaultRelease = "4.19.0-gvisor" + + // rdmaRelease is the release advertised instead when RDMA support is + // enabled. It must be >= 5.12 so RDMA userspace enables dmabuf-based + // memory registration for GPUDirect: aws-ofi-nccl (the EFA NCCL + // transport) gates ibv_reg_dmabuf_mr() on the uname(2) release, and the + // RDMA dmabuf uverbs ioctls landed upstream in Linux 5.12. Without this + // the EFA provider falls back to registering raw CUDA VAs, which the + // sandbox cannot pin. Advertised only in RDMA sandboxes to limit the + // bump's blast radius; a later change can make it the default. + rdmaRelease = "5.15.0-gvisor" // LinuxVersion is the version info advertised by gVisor. LinuxVersion = "#1 SMP Sun Jan 10 15:06:54 PST 2016" ) + +// release is the advertised release; see UseRDMARelease. +var release = defaultRelease + +// LinuxRelease returns the Linux release version number advertised by gVisor. +func LinuxRelease() string { + return release +} + +// UseRDMARelease switches the advertised release to rdmaRelease. It must only +// be called during early boot. +func UseRDMARelease() { + release = rdmaRelease +} diff --git a/pkg/sentry/mm/syscalls.go b/pkg/sentry/mm/syscalls.go index 46c32ecd40f..4dbe6dec756 100644 --- a/pkg/sentry/mm/syscalls.go +++ b/pkg/sentry/mm/syscalls.go @@ -1539,3 +1539,18 @@ func (mm *MemoryManager) FindVMAByName(ar hostarch.AddrRange, name string) (host } return 0, 0, fmt.Errorf("could not find %q in %s", name, ar) } + +// FindVMARange returns the address range of the vma containing addr, or an +// error if addr is not mapped. The RDMA proxy uses it to bound a +// driver-private DMA buffer (work queue / doorbell) whose backing region the +// guest describes by start address only, with no explicit length. +func (mm *MemoryManager) FindVMARange(addr hostarch.Addr) (hostarch.AddrRange, error) { + mm.mappingMu.RLock() + defer mm.mappingMu.RUnlock() + + vseg := mm.vmas.FindSegment(addr) + if !vseg.Ok() { + return hostarch.AddrRange{}, linuxerr.EFAULT + } + return vseg.Range(), nil +} diff --git a/pkg/sentry/syscalls/linux/linux64.go b/pkg/sentry/syscalls/linux/linux64.go index 10e38790f74..0001a1fd69a 100644 --- a/pkg/sentry/syscalls/linux/linux64.go +++ b/pkg/sentry/syscalls/linux/linux64.go @@ -22,24 +22,14 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/kernel" - "gvisor.dev/gvisor/pkg/sentry/kernel/version" "gvisor.dev/gvisor/pkg/sentry/syscalls" ) // AMD64 is a table of Linux amd64 syscall API with the corresponding syscall // numbers from Linux 4.4. var AMD64 = &kernel.SyscallTable{ - OS: abi.Linux, - Arch: arch.AMD64, - Version: kernel.Version{ - // Version 4.4 is chosen as a stable, longterm version of Linux, which - // guides the interface provided by this syscall table. The build - // version is that for a clean build with default kernel config, at 5 - // minutes after v4.4 was tagged. - Sysname: version.LinuxSysname, - Release: version.LinuxRelease, - Version: version.LinuxVersion, - }, + OS: abi.Linux, + Arch: arch.AMD64, AuditNumber: linux.AUDIT_ARCH_X86_64, Table: map[uintptr]kernel.Syscall{ 0: syscalls.SupportedPoint("read", Read, PointRead), @@ -413,13 +403,8 @@ var AMD64 = &kernel.SyscallTable{ // ARM64 is a table of Linux arm64 syscall API with the corresponding syscall // numbers from Linux 4.4. var ARM64 = &kernel.SyscallTable{ - OS: abi.Linux, - Arch: arch.ARM64, - Version: kernel.Version{ - Sysname: version.LinuxSysname, - Release: version.LinuxRelease, - Version: version.LinuxVersion, - }, + OS: abi.Linux, + Arch: arch.ARM64, AuditNumber: linux.AUDIT_ARCH_AARCH64, Table: map[uintptr]kernel.Syscall{ 0: syscalls.PartiallySupported("io_setup", IoSetup, "Generally supported with exceptions. User ring optimizations are not implemented.", []string{"gvisor.dev/issue/204"}), diff --git a/pkg/sentry/syscalls/linux/sys_utsname.go b/pkg/sentry/syscalls/linux/sys_utsname.go index 40eb6912f57..32608ebe695 100644 --- a/pkg/sentry/syscalls/linux/sys_utsname.go +++ b/pkg/sentry/syscalls/linux/sys_utsname.go @@ -19,20 +19,19 @@ import ( "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/kernel/version" ) // Uname implements linux syscall uname. func Uname(t *kernel.Task, sysno uintptr, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) { - version := t.SyscallTable().Version - uts := t.UTSNamespace() // Fill in structure fields. var u linux.UtsName - copy(u.Sysname[:], version.Sysname) + copy(u.Sysname[:], version.LinuxSysname) copy(u.Nodename[:], uts.HostName()) - copy(u.Release[:], version.Release) - copy(u.Version[:], version.Version) + copy(u.Release[:], version.LinuxRelease()) + copy(u.Version[:], version.LinuxVersion) // build tag above. switch t.SyscallTable().Arch { case arch.AMD64: diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index b67342bfb8f..1b9e65b30a0 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -35,6 +35,7 @@ go_library( ], deps = [ "//pkg/abi", + "//pkg/abi/ib", "//pkg/abi/linux", "//pkg/abi/nvgpu", "//pkg/bpf", @@ -65,6 +66,9 @@ go_library( "//pkg/sentry/devices/memdev", "//pkg/sentry/devices/nvproxy", "//pkg/sentry/devices/nvproxy/nvconf", + "//pkg/sentry/devices/rdmaproxy", + "//pkg/sentry/devices/rdmaproxy/cxproxy", + "//pkg/sentry/devices/rdmaproxy/efaproxy", "//pkg/sentry/devices/tpuproxy", "//pkg/sentry/devices/tpuproxy/vfio", "//pkg/sentry/devices/ttydev", @@ -91,6 +95,7 @@ go_library( "//pkg/sentry/kernel/auth", "//pkg/sentry/limits", "//pkg/sentry/loader", + "//pkg/sentry/kernel/version", "//pkg/sentry/pgalloc", "//pkg/sentry/platform", "//pkg/sentry/platform/platforms", @@ -100,6 +105,7 @@ go_library( "//pkg/sentry/seccheck/sinks/remote", "//pkg/sentry/socket/hostinet", "//pkg/sentry/socket/netfilter", + "//pkg/sentry/syscalls/linux", "//pkg/sentry/socket/netlink", "//pkg/sentry/socket/netlink/netfilter", "//pkg/sentry/socket/netlink/route", diff --git a/runsc/boot/filter/config/BUILD b/runsc/boot/filter/config/BUILD index 6158deb3dae..42fd484b86c 100644 --- a/runsc/boot/filter/config/BUILD +++ b/runsc/boot/filter/config/BUILD @@ -33,6 +33,7 @@ go_library( "//pkg/seccomp/precompiledseccomp", "//pkg/sentry/devices/nvproxy", "//pkg/sentry/devices/nvproxy/nvconf", + "//pkg/sentry/devices/rdmaproxy", "//pkg/sentry/devices/tpuproxy", "//pkg/sentry/platform", "//pkg/sentry/platform/platforms", diff --git a/runsc/boot/filter/config/config.go b/runsc/boot/filter/config/config.go index 9a9cd28c9bf..93836896803 100644 --- a/runsc/boot/filter/config/config.go +++ b/runsc/boot/filter/config/config.go @@ -27,6 +27,7 @@ import ( "gvisor.dev/gvisor/pkg/seccomp/precompiledseccomp" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy" "gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/socket/plugin" @@ -42,6 +43,7 @@ type Options struct { NVProxy bool NVProxyCaps nvconf.DriverCaps TPUProxy bool + RDMAProxy bool ControllerFD uint32 CgoEnabled bool PluginNetwork bool @@ -71,6 +73,7 @@ func (opt Options) ConfigKey() string { fmt.Fprintf(&sb, "NVProxy=%t ", opt.NVProxy) fmt.Fprintf(&sb, "NVProxyCaps=%v ", opt.NVProxyCaps) fmt.Fprintf(&sb, "TPUProxy=%t ", opt.TPUProxy) + fmt.Fprintf(&sb, "RDMAProxy=%t ", opt.RDMAProxy) fmt.Fprintf(&sb, "CgoEnabled=%t ", opt.CgoEnabled) fmt.Fprintf(&sb, "PluginNetwork=%t ", opt.PluginNetwork) return strings.TrimSpace(sb.String()) @@ -102,6 +105,9 @@ func Warnings(opt Options) []string { if opt.TPUProxy { warnings = append(warnings, "TPU device proxy enabled: syscall filters less restrictive!") } + if opt.RDMAProxy { + warnings = append(warnings, "RDMA device proxy enabled: syscall filters less restrictive!") + } if opt.CgoEnabled { warnings = append(warnings, "CGO enabled: syscall filters less restrictive!") } @@ -155,6 +161,9 @@ func rules(opt Options, vars precompiledseccomp.Values) (seccomp.SyscallRules, s if opt.TPUProxy { s.Merge(tpuproxy.Filters()) } + if opt.RDMAProxy { + s.Merge(rdmaproxy.Filters()) + } if opt.CgoEnabled { s.Merge(cgoFilters()) } diff --git a/runsc/boot/filter/config/config_precompiled.go b/runsc/boot/filter/config/config_precompiled.go index 2872f1ca8fd..f5c8915bfdb 100644 --- a/runsc/boot/filter/config/config_precompiled.go +++ b/runsc/boot/filter/config/config_precompiled.go @@ -111,6 +111,15 @@ func optionsToPrecompile() ([]Options, error) { tpuProxyNo.TPUProxy = false return []Options{tpuProxyYes, tpuProxyNo}, nil }, + + // Expand RDMAProxy vs not. + func(opt Options) ([]Options, error) { + rdmaProxyYes := opt + rdmaProxyYes.RDMAProxy = true + rdmaProxyNo := opt + rdmaProxyNo.RDMAProxy = false + return []Options{rdmaProxyYes, rdmaProxyNo}, nil + }, } { var newOpts []Options for _, opt := range opts { diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index c2f2575e078..e3880c7b743 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -43,6 +43,8 @@ import ( "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy/cxproxy" + "gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy/efaproxy" "gvisor.dev/gvisor/pkg/sentry/fdimport" cgroup2fs "gvisor.dev/gvisor/pkg/sentry/fsimpl/cgroup2fs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/host" @@ -51,6 +53,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/kernel/version" "gvisor.dev/gvisor/pkg/sentry/loader" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/platform" @@ -539,6 +542,11 @@ func New(args Args) (*Loader, error) { if specutils.NVProxyEnabled(args.Spec, args.Conf) { nvproxy.Init() } + if specutils.RDMAEnabled(args.Spec, args.Conf) { + cxproxy.Init() + efaproxy.Init() + version.UseRDMARelease() + } eid := execID{cid: args.ID} l := &Loader{ @@ -756,7 +764,7 @@ func New(args Args) (*Loader, error) { return nil, fmt.Errorf("initializing kernel: %w", err) } - if err := registerFilesystems(l.k, &l.root); err != nil { + if err := registerFilesystems(l.k, &l.root, l.rdmaSysfs); err != nil { return nil, fmt.Errorf("registering filesystems: %w", err) } @@ -1094,6 +1102,7 @@ func (l *Loader) installSeccompFilters() error { NVProxy: nvproxyEnabled, NVProxyCaps: nvproxyCaps, TPUProxy: specutils.TPUProxyEnabled(l.root.spec, l.root.conf), + RDMAProxy: specutils.RDMAEnabled(l.root.spec, l.root.conf), ControllerFD: uint32(l.ctrl.srv.FD()), CgoEnabled: config.CgoEnabled, PluginNetwork: l.root.conf.Network == config.NetworkPlugin, diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 1212a699007..8bcd71403d4 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -26,6 +26,7 @@ import ( "strings" specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/abi/ib" "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/abi/nvgpu" "gvisor.dev/gvisor/pkg/cleanup" @@ -41,6 +42,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/devices/memdev" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy" "gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy" "gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy/vfio" "gvisor.dev/gvisor/pkg/sentry/devices/ttydev" @@ -118,7 +120,7 @@ func cgroupfsMemoryDefaults(memoryLimit uint64) map[string]int64 { } } -func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { +func registerFilesystems(k *kernel.Kernel, info *containerInfo, rdmaSnapshot *rdma.Snapshot) error { ctx := k.SupervisorContext() vfsObj := k.VFS() @@ -193,6 +195,71 @@ func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { return err } + if err := rdmaproxyRegisterDevices(info, vfsObj, rdmaSnapshot); err != nil { + return err + } + + return nil +} + +// rdmaproxyRegisterDevices registers a proxied VFS char device for each +// /dev/infiniband/uverbs* device in the spec, so the sandbox can drive RDMA +// verbs against the host. Devices are registered at the fixed uverbs +// char-device major (see rdmaproxy.Register), which matches the major the +// guest device node and its /sys .../dev entry carry. The vendor driver +// plug-in is selected per device from the PCI driver name in the host sysfs +// snapshot (e.g. mlx5_core -> cxproxy). +func rdmaproxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem, snapshot *rdma.Snapshot) error { + if snapshot == nil || !specutils.RDMAEnabled(info.spec, info.conf) { + return nil + } + // Map host uverbs minor -> PCI driver name, resolved from the leaf PCI + // node's uevent in the snapshot. + leafUevent := make(map[string]string, len(snapshot.PCINodes)) + for i := range snapshot.PCINodes { + leafUevent[snapshot.PCINodes[i].Path] = snapshot.PCINodes[i].Attrs["uevent"] + } + driverByMinor := make(map[uint32]string) + for i := range snapshot.Devices { + dev := &snapshot.Devices[i] + // Snapshot attribute values are verbatim file contents; trim before + // parsing "major:minor". + devStr := strings.TrimSpace(dev.Dev) + sep := strings.IndexByte(devStr, ':') + if sep < 0 { + continue + } + minor64, err := strconv.ParseUint(devStr[sep+1:], 10, 32) + if err != nil { + continue + } + for _, line := range strings.Split(leafUevent[dev.LeafPCI], "\n") { + if driver, ok := strings.CutPrefix(line, "DRIVER="); ok { + driverByMinor[uint32(minor64)] = driver + break + } + } + } + for _, devSpec := range info.spec.Linux.Devices { + if !strings.HasPrefix(devSpec.Path, "/dev/infiniband/uverbs") { + continue + } + // We expect the host uverbs devices to carry the fixed InfiniBand + // uverbs major (IB_UVERBS_MAJOR). A device with any other major is one + // the host assigned a dynamically-allocated number (happen with a host + // with >32 RDMA devices), which we currently do not support. + if uint32(devSpec.Major) != ib.IB_UVERBS_MAJOR { + return fmt.Errorf("rdma: uverbs device %s has char-device major %d, want %d: devices with a dynamically-allocated major are not supported", devSpec.Path, devSpec.Major, ib.IB_UVERBS_MAJOR) + } + minor := uint32(devSpec.Minor) + devName := filepath.Base(devSpec.Path) + driverName := driverByMinor[minor] + if err := rdmaproxy.Register(vfsObj, devName, minor, driverName, snapshot.VerbsABIVersion); err != nil { + log.Warningf("rdma: register %s: %v", devSpec.Path, err) + continue + } + log.Infof("rdma: registered %s minor=%d driver=%q", devSpec.Path, minor, driverName) + } return nil } diff --git a/runsc/cmd/sandboxsetup/gofer_mount.go b/runsc/cmd/sandboxsetup/gofer_mount.go index 3dc850b8e39..e3091adee49 100644 --- a/runsc/cmd/sandboxsetup/gofer_mount.go +++ b/runsc/cmd/sandboxsetup/gofer_mount.go @@ -506,6 +506,13 @@ func ShouldExposeTpuDevice(path string) bool { return valid || ShouldExposeVFIODevice(path) } +// ShouldExposeRDMADevice returns true if path refers to an RDMA uverbs device. +// +// Precondition: rdmaproxy is enabled. +func ShouldExposeRDMADevice(path string) bool { + return strings.HasPrefix(path, "/dev/infiniband/uverbs") +} + // SetupDev mounts devices from the OCI spec into the gofer's /dev directory. func SetupDev(spec *specs.Spec, conf *config.Config, root, procPath string) error { if err := os.MkdirAll(filepath.Join(root, "dev"), 0777); err != nil { @@ -517,9 +524,11 @@ func SetupDev(spec *specs.Spec, conf *config.Config, root, procPath string) erro } nvproxyEnabled := specutils.NVProxyEnabled(spec, conf) tpuproxyEnabled := specutils.TPUProxyEnabled(spec, conf) + rdmaproxyEnabled := specutils.RDMAEnabled(spec, conf) for _, dev := range spec.Linux.Devices { shouldMount := (nvproxyEnabled && ShouldExposeNvidiaDevice(dev.Path)) || - (tpuproxyEnabled && ShouldExposeTpuDevice(dev.Path)) + (tpuproxyEnabled && ShouldExposeTpuDevice(dev.Path)) || + (rdmaproxyEnabled && ShouldExposeRDMADevice(dev.Path)) if !shouldMount { continue } diff --git a/runsc/container/container.go b/runsc/container/container.go index 4ee19ddbf89..8a6c86c28fd 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -1333,7 +1333,7 @@ func (c *Container) waitForStopped() error { // shouldCreateDeviceGofer indicates whether a device gofer connection should // be created. func shouldCreateDeviceGofer(spec *specs.Spec, conf *config.Config) bool { - return specutils.GPUFunctionalityRequested(spec, conf) || specutils.TPUFunctionalityRequested(spec, conf) + return specutils.GPUFunctionalityRequested(spec, conf) || specutils.TPUFunctionalityRequested(spec, conf) || specutils.RDMAEnabled(spec, conf) } // shouldSpawnGofer indicates whether the gofer process should be spawned. diff --git a/runsc/specutils/nvidia.go b/runsc/specutils/nvidia.go index 5ffa85a76a3..03392f66334 100644 --- a/runsc/specutils/nvidia.go +++ b/runsc/specutils/nvidia.go @@ -186,6 +186,12 @@ func NVProxyDriverCapsAllowed(conf *config.Config) (nvconf.DriverCaps, error) { if hasAll { allowedDriverCaps |= nvconf.AllContainerDriverCaps } + // GPUDirect RDMA requires the privileged CapRDMA capability to export GPU + // memory to a dma-buf fd. There is no container-facing driver-capability + // flag for it, so enable it implicitly whenever RDMA passthrough is on. + if conf.RDMAProxy { + allowedDriverCaps |= nvconf.CapRDMA + } return allowedDriverCaps, nil } diff --git a/website/BUILD b/website/BUILD index 3f5e588b394..ba1867ab5f0 100644 --- a/website/BUILD +++ b/website/BUILD @@ -171,6 +171,7 @@ docs( "//g3doc/user_guide:observability", "//g3doc/user_guide:platforms", "//g3doc/user_guide:production", + "//g3doc/user_guide:rdma", "//g3doc/user_guide:rootfs_snapshot", "//g3doc/user_guide:rootless", "//g3doc/user_guide:runtime_monitoring",