diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp index 5c3fd4581..37aa9327f 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp @@ -37,24 +37,13 @@ using cudaq::qec::decoding::config::DecoderTransport; // Constructors // --------------------------------------------------------------------------- -/// gpu_roce runs the whole pipeline -- rings, dispatch scheduler, device-side -/// graph fire -- on ONE GPU: the one the FPGA/NIC is affine to -/// (HOLOLINK_GPU_ID). A decoder pinned elsewhere would split graph capture -/// and graph launch across devices, which CUDA graphs cannot do. Both knobs -/// name the same topology fact, so they must agree. -int reconcile_gpu_roce_device(std::optional env_gpu_id, int decoder_pin) { - if (env_gpu_id && *env_gpu_id < 0) - throw std::runtime_error("HOLOLINK_GPU_ID must be >= 0 (got " + - std::to_string(*env_gpu_id) + ")"); - if (env_gpu_id && decoder_pin >= 0 && *env_gpu_id != decoder_pin) - throw std::runtime_error( - "gpu_roce device conflict: HOLOLINK_GPU_ID=" + - std::to_string(*env_gpu_id) + " but the decoder is pinned to " + - std::to_string(decoder_pin) + - " (cuda_device_id). The FPGA-affine GPU and the decoder pin must be " - "the same device."); - if (env_gpu_id) - return *env_gpu_id; +/// Resolve the CUDA device a decode pipeline runs on from the decoder's +/// cuda_device_id pin; an unpinned decoder (-1) defaults to device 0. The +/// gpu_roce path relies on this to place its rings, dispatch scheduler, and +/// device-side graph fire on the one GPU the FPGA/NIC is affine to -- CUDA +/// graphs cannot split capture and launch across devices, so the decoder must +/// be pinned to that device. +int resolve_decode_device(int decoder_pin) { return decoder_pin >= 0 ? decoder_pin : 0; } @@ -64,9 +53,9 @@ DecodingServer::make_transport(DecoderTransport transport_type, switch (transport_type) { case DecoderTransport::gpu_roce: // gpu_roce lives in the cudaq-qec-decoding-server-gpuroce component, - // reached through the weak factory. The device reconciliation (env - // HOLOLINK_GPU_ID vs the decoder's cuda_device_id pin) happens inside the - // factory, where GpuRoceConfig lives; we just thread the pin to it. + // reached through the weak factory. The device is the decoder's + // cuda_device_id pin, resolved inside the factory where GpuRoceConfig + // lives; we just thread the pin to it. if (cudaqx_qec_make_gpu_roce_transceiver) return std::unique_ptr( cudaqx_qec_make_gpu_roce_transceiver(pinned_cuda_device)); @@ -103,7 +92,7 @@ DecodingServer::DecodingServer(const std::string &config_yaml) { const auto transport_type = registry_.required_transport(); // gpu_roce must run on the GPU the FPGA/NIC is affine to; when exactly one // session is booting, pass its decoder's cuda_device_id so the factory can - // reconcile it against HOLOLINK_GPU_ID. + // place the transport on that device. const auto &boot_sessions = registry_.sessions(); const int pinned_cuda_device = boot_sessions.size() == 1 diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h index fbc235bd7..4902e256a 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h +++ b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h @@ -15,18 +15,15 @@ #include #include -#include #include #include #include namespace cudaq::qec::decoding_server { -/// Resolve the single GPU a gpu_roce pipeline runs on from the two knobs that -/// can name it: HOLOLINK_GPU_ID (FPGA/NIC affinity; nullopt when unset) and -/// the decoder's cuda_device_id (-1 when unpinned). Throws when both are set -/// and disagree; unset env defers to the pin; neither set -> 0. -int reconcile_gpu_roce_device(std::optional env_gpu_id, int decoder_pin); +/// Resolve the CUDA device a decode pipeline runs on from the decoder's +/// cuda_device_id (-1 when unpinned). An unpinned decoder defaults to device 0. +int resolve_decode_device(int decoder_pin); /// Maps function_id → non-owning ITransceiver pointer. /// Ownership lives in DecodingServer::owned_transports_. diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp index a02dbdd9f..33beca6a5 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp @@ -13,33 +13,17 @@ #include "cudaq/qec/logger.h" #include -#include #include #include -#include #include -#include #include namespace cudaq::qec::decoding_server { namespace { -std::optional env_int_optional(const char *name) { - const char *value = std::getenv(name); - if (!value || !*value) - return std::nullopt; - try { - return std::stoi(value); - } catch (const std::exception &) { - throw std::runtime_error(std::string("invalid ") + name + - " value: " + value); - } -} - void set_graph_capture_device(const cudaq::qec::decoder &decoder) { - const int device = reconcile_gpu_roce_device( - env_int_optional("HOLOLINK_GPU_ID"), decoder.get_cuda_device_id()); + const int device = resolve_decode_device(decoder.get_cuda_device_id()); cudaq::qec::detail_affinity::set_cuda_device_for_decode(device); if (device >= 0) CUDA_QEC_INFO( diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp index 8f2e05aec..651290121 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp @@ -13,15 +13,15 @@ // component WHOLE_ARCHIVE: the sole reference to this symbol is weak, which // does not pull archive members on its own. -#include "DecodingServer.h" // reconcile_gpu_roce_device (core symbol) +#include "DecodingServer.h" // resolve_decode_device (core symbol) #include "GpuRoceTransceiver.h" extern "C" cudaq::qec::decoding_server::ITransceiver * cudaqx_qec_make_gpu_roce_transceiver(int pinned_cuda_device) { using namespace cudaq::qec::decoding_server; - // Reconcile the FPGA-affine GPU (HOLOLINK_GPU_ID) with the decoder's pin - // here, inside the component, where GpuRoceConfig is visible. + // The gpu_roce device is the decoder's cuda_device_id pin; resolve it here, + // inside the component, where GpuRoceConfig is visible. auto cfg = GpuRoceConfig::from_env(); - cfg.gpu_id = reconcile_gpu_roce_device(cfg.gpu_id_env, pinned_cuda_device); + cfg.gpu_id = resolve_decode_device(pinned_cuda_device); return new GpuRoceTransceiver(cfg); } diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp index 15e8f927e..df6ae4fba 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp @@ -119,9 +119,9 @@ GpuRoceConfig GpuRoceConfig::from_env() { c.device_name = env_str("HOLOLINK_DEVICE"); c.peer_ip = env_str("HOLOLINK_PEER_IP"); c.remote_qp = env_u32("HOLOLINK_REMOTE_QP", 0); - if (std::getenv("HOLOLINK_GPU_ID")) - c.gpu_id_env = env_int("HOLOLINK_GPU_ID", 0); - c.gpu_id = c.gpu_id_env.value_or(0); + // gpu_id is not read from the environment: the device is the decoder's + // cuda_device_id, resolved by resolve_decode_device() at transport + // creation. c.frame_size = env_size("HOLOLINK_FRAME_SIZE", 384); c.page_size = env_size("HOLOLINK_PAGE_SIZE", 0); // 0 → derived below c.num_pages = env_size("HOLOLINK_NUM_PAGES", 64); diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h index dbdb88ff0..89200f81a 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h +++ b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -32,21 +31,20 @@ struct cudaq_dispatch_graph_context; namespace cudaq::qec::decoding_server { -/// Runtime configuration for GpuRoceTransceiver. All fields are read from -/// environment variables so that the server can be reconfigured without a -/// rebuild. +/// Runtime configuration for GpuRoceTransceiver. Transport fields are read +/// from environment variables so that the server can be reconfigured without a +/// rebuild; gpu_id is the exception -- it is the decoder's cuda_device_id, +/// filled in at transport creation. struct GpuRoceConfig { std::string device_name; ///< HOLOLINK_DEVICE (IB netdev, e.g. "mlx5_0") uint32_t remote_qp{0}; ///< HOLOLINK_REMOTE_QP (FPGA/emulator QP number) - int gpu_id{0}; ///< HOLOLINK_GPU_ID - /// Set iff HOLOLINK_GPU_ID was present in the environment (the FPGA/NIC - /// affinity is a topology fact; absence defers to the decoder's pin). - std::optional gpu_id_env; - size_t frame_size{384}; ///< HOLOLINK_FRAME_SIZE (max RPC frame bytes) - size_t page_size{0}; ///< HOLOLINK_PAGE_SIZE (0 → derived from frame_size) - size_t num_pages{64}; ///< HOLOLINK_NUM_PAGES (ring depth) - std::string peer_ip; ///< HOLOLINK_PEER_IP (FPGA/emulator IPv4) - int reserved_sms{2}; ///< HOLOLINK_RESERVED_SMS (SMs for Hololink RX/TX) + int gpu_id{0}; ///< FPGA-affine GPU; set from the decoder's + ///< cuda_device_id by resolve_decode_device() + size_t frame_size{384}; ///< HOLOLINK_FRAME_SIZE (max RPC frame bytes) + size_t page_size{0}; ///< HOLOLINK_PAGE_SIZE (0 → derived from frame_size) + size_t num_pages{64}; ///< HOLOLINK_NUM_PAGES (ring depth) + std::string peer_ip; ///< HOLOLINK_PEER_IP (FPGA/emulator IPv4) + int reserved_sms{2}; ///< HOLOLINK_RESERVED_SMS (SMs for Hololink RX/TX) static GpuRoceConfig from_env(); }; diff --git a/libs/qec/unittests/test_decoding_server_core.cpp b/libs/qec/unittests/test_decoding_server_core.cpp index 76115afdb..9fc9c5514 100644 --- a/libs/qec/unittests/test_decoding_server_core.cpp +++ b/libs/qec/unittests/test_decoding_server_core.cpp @@ -261,29 +261,12 @@ TEST(RpcDispatcherTest, ConvertsHandlerExceptionsToErrorResponses) { expect_status(transport, RpcStatus::INTERNAL_ERROR); } -TEST(GpuRoceDeviceReconcile, BothUnsetDefaultsToZero) { - EXPECT_EQ( - cudaq::qec::decoding_server::reconcile_gpu_roce_device(std::nullopt, -1), - 0); +TEST(ResolveDecodeDevice, UnpinnedDefaultsToZero) { + EXPECT_EQ(cudaq::qec::decoding_server::resolve_decode_device(-1), 0); } -TEST(GpuRoceDeviceReconcile, EnvOnlyWins) { - EXPECT_EQ(cudaq::qec::decoding_server::reconcile_gpu_roce_device(2, -1), 2); -} - -TEST(GpuRoceDeviceReconcile, PinOnlyWins) { - EXPECT_EQ( - cudaq::qec::decoding_server::reconcile_gpu_roce_device(std::nullopt, 3), - 3); -} - -TEST(GpuRoceDeviceReconcile, AgreementPasses) { - EXPECT_EQ(cudaq::qec::decoding_server::reconcile_gpu_roce_device(1, 1), 1); -} - -TEST(GpuRoceDeviceReconcile, ConflictThrows) { - EXPECT_THROW(cudaq::qec::decoding_server::reconcile_gpu_roce_device(0, 2), - std::runtime_error); +TEST(ResolveDecodeDevice, PinSelectsDevice) { + EXPECT_EQ(cudaq::qec::decoding_server::resolve_decode_device(3), 3); } TEST(SetCudaDeviceForDecode, UnpinnedIsNoOp) { @@ -336,11 +319,6 @@ TEST(DecodingSessionPinHandshake, UnhonorablePinFailsStartWorker) { EXPECT_FALSE(session->worker.joinable()); } -TEST(GpuRoceDeviceReconcile, NegativeEnvThrows) { - EXPECT_THROW(cudaq::qec::decoding_server::reconcile_gpu_roce_device(-1, -1), - std::runtime_error); -} - TEST(DecodingSessionPinHandshake, PinnedWorkerStartsAndServes) { // start_worker() must resolve the pin handshake (throwing on failure per // its contract) and leave a live worker serving items. diff --git a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh index 91d6b9ba9..47cd35759 100755 --- a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh +++ b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh @@ -1009,7 +1009,6 @@ start_server() { HOLOLINK_REMOTE_QP="$((remote_qp))" \ HOLOLINK_FRAME_SIZE="$PAGE_SIZE" \ HOLOLINK_NUM_PAGES="$GPU_ROCE_NUM_PAGES" \ - HOLOLINK_GPU_ID="$GPU_ID" \ "$SERVER_BIN" \ --config="$CONFIG_FILE" \ --transport=gpu_roce \