diff --git a/CMakeLists.txt b/CMakeLists.txt index 06862ae58..e1f9ce0a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,7 +88,7 @@ set(PACKAGE_VERSION "65.0") # When this is changed the values in these files need changing too: # debian/control # debian/libibverbs1.symbols -set(IBVERBS_PABI_VERSION "64") +set(IBVERBS_PABI_VERSION "65") set(IBVERBS_PROVIDER_SUFFIX "-rdmav${IBVERBS_PABI_VERSION}.so") #------------------------- diff --git a/debian/control b/debian/control index b7a731abe..736fd1c8f 100644 --- a/debian/control +++ b/debian/control @@ -151,7 +151,7 @@ Section: libs Pre-Depends: ${misc:Pre-Depends} Depends: adduser, ${misc:Depends}, ${shlibs:Depends} Recommends: ibverbs-providers -Breaks: ibverbs-providers (<< 64~) +Breaks: ibverbs-providers (<< 65~) Description: Library for direct userspace use of RDMA (InfiniBand/iWARP) libibverbs is a library that allows userspace processes to use RDMA "verbs" as described in the InfiniBand Architecture Specification and diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 3697696ab..40e627d5d 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols @@ -188,6 +188,7 @@ libefa.so.1 ibverbs-providers #MINVER# EFA_1.4@EFA_1.4 59 EFA_1.5@EFA_1.5 63 EFA_1.6@EFA_1.6 64 + EFA_1.7@EFA_1.7 65 efadv_create_driver_qp@EFA_1.0 24 efadv_create_qp_ex@EFA_1.1 26 efadv_query_device@EFA_1.1 26 @@ -200,6 +201,7 @@ libefa.so.1 ibverbs-providers #MINVER# efadv_get_max_sq_depth@EFA_1.5 63 efadv_get_max_rq_depth@EFA_1.5 63 efadv_qp_from_ibv_qp_ex@EFA_1.6 64 + efadv_create_comp_cntr@EFA_1.7 65 libhns.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev HNS_1.0@HNS_1.0 51 diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols index 6f90cffde..3b7494183 100644 --- a/debian/libibverbs1.symbols +++ b/debian/libibverbs1.symbols @@ -16,7 +16,8 @@ libibverbs.so.1 libibverbs1 #MINVER# IBVERBS_1.16@IBVERBS_1.16 62 IBVERBS_1.17@IBVERBS_1.17 64 IBVERBS_1.18@IBVERBS_1.18 65 - (symver)IBVERBS_PRIVATE_64 64 + (symver)IBVERBS_PRIVATE_65 65 + _ibv_query_comp_cntr_caps@IBVERBS_1.18 65 _ibv_query_gid_ex@IBVERBS_1.11 32 _ibv_query_gid_table@IBVERBS_1.11 32 ibv_ack_async_event@IBVERBS_1.0 1.1.6 @@ -39,6 +40,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_create_ah@IBVERBS_1.1 1.1.6 ibv_create_ah_from_wc@IBVERBS_1.1 1.1.6 ibv_create_comp_channel@IBVERBS_1.0 1.1.6 + ibv_create_comp_cntr@IBVERBS_1.18 65 ibv_create_cq@IBVERBS_1.0 1.1.6 ibv_create_cq@IBVERBS_1.1 1.1.6 ibv_create_qp@IBVERBS_1.0 1.1.6 @@ -53,6 +55,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_destroy_ah@IBVERBS_1.0 1.1.6 ibv_destroy_ah@IBVERBS_1.1 1.1.6 ibv_destroy_comp_channel@IBVERBS_1.0 1.1.6 + ibv_destroy_comp_cntr@IBVERBS_1.18 65 ibv_destroy_cq@IBVERBS_1.0 1.1.6 ibv_destroy_cq@IBVERBS_1.1 1.1.6 ibv_destroy_qp@IBVERBS_1.0 1.1.6 @@ -87,6 +90,8 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_import_dm@IBVERBS_1.13 35 ibv_import_mr@IBVERBS_1.10 31 ibv_import_pd@IBVERBS_1.10 31 + ibv_inc_comp_cntr@IBVERBS_1.18 65 + ibv_inc_err_comp_cntr@IBVERBS_1.18 65 ibv_init_ah_from_wc@IBVERBS_1.1 1.1.6 ibv_is_fork_initialized@IBVERBS_1.13 35 ibv_modify_qp@IBVERBS_1.0 1.1.6 @@ -97,6 +102,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_open_device@IBVERBS_1.0 1.1.6 ibv_open_device@IBVERBS_1.1 1.1.6 ibv_port_state_str@IBVERBS_1.1 1.1.6 + ibv_qp_attach_comp_cntr@IBVERBS_1.18 65 ibv_qp_to_qp_ex@IBVERBS_1.6 24 ibv_query_device@IBVERBS_1.0 1.1.6 ibv_query_device@IBVERBS_1.1 1.1.6 @@ -115,6 +121,8 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_query_srq@IBVERBS_1.1 1.1.6 ibv_rate_to_mbps@IBVERBS_1.1 1.1.8 ibv_rate_to_mult@IBVERBS_1.0 1.1.6 + ibv_read_comp_cntr@IBVERBS_1.18 65 + ibv_read_err_comp_cntr@IBVERBS_1.18 65 ibv_read_sysfs_file@IBVERBS_1.0 1.1.6 ibv_reg_buf_mr@IBVERBS_1.17 64 ibv_reg_dmabuf_mr@IBVERBS_1.12 34 @@ -128,7 +136,9 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_resize_cq@IBVERBS_1.0 1.1.6 ibv_resize_cq@IBVERBS_1.1 1.1.6 ibv_resolve_eth_l2_from_gid@IBVERBS_1.1 1.2.0 + ibv_set_comp_cntr@IBVERBS_1.18 65 ibv_set_ece@IBVERBS_1.10 31 + ibv_set_err_comp_cntr@IBVERBS_1.18 65 ibv_unimport_dm@IBVERBS_1.13 35 ibv_unimport_mr@IBVERBS_1.10 31 ibv_unimport_pd@IBVERBS_1.10 31 diff --git a/kernel-headers/rdma/efa-abi.h b/kernel-headers/rdma/efa-abi.h index d5c18f8de..c79b54aad 100644 --- a/kernel-headers/rdma/efa-abi.h +++ b/kernel-headers/rdma/efa-abi.h @@ -133,6 +133,7 @@ enum { EFA_QUERY_DEVICE_CAPS_RDMA_WRITE = 1 << 5, EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV = 1 << 6, EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM = 1 << 7, + EFA_QUERY_DEVICE_CAPS_COMP_CNTR = 1 << 8, }; struct efa_ibv_ex_query_device_resp { @@ -163,4 +164,9 @@ enum efa_mr_methods { EFA_IB_METHOD_MR_QUERY = (1U << UVERBS_ID_NS_SHIFT), }; +enum efa_comp_cntr_create_attrs { + EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER = (1U << UVERBS_ID_NS_SHIFT), + EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, +}; + #endif /* EFA_ABI_USER_H */ diff --git a/kernel-headers/rdma/ib_user_ioctl_cmds.h b/kernel-headers/rdma/ib_user_ioctl_cmds.h index 839835bd4..6a3d59d03 100644 --- a/kernel-headers/rdma/ib_user_ioctl_cmds.h +++ b/kernel-headers/rdma/ib_user_ioctl_cmds.h @@ -57,6 +57,7 @@ enum uverbs_default_objects { UVERBS_OBJECT_ASYNC_EVENT, UVERBS_OBJECT_DMAH, UVERBS_OBJECT_DMABUF, + UVERBS_OBJECT_COMP_CNTR, }; enum { @@ -75,6 +76,7 @@ enum uverbs_methods_device { UVERBS_METHOD_QUERY_GID_TABLE, UVERBS_METHOD_QUERY_GID_ENTRY, UVERBS_METHOD_QUERY_PORT_SPEED, + UVERBS_METHOD_QUERY_COMP_CNTR_CAPS, }; enum uverbs_attrs_invoke_write_cmd_attr_ids { @@ -93,6 +95,12 @@ enum uverbs_attrs_query_port_speed_cmd_attr_ids { UVERBS_ATTR_QUERY_PORT_SPEED_RESP, }; +enum uverbs_attrs_query_comp_cntr_caps_attr_ids { + UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, +}; + enum uverbs_attrs_get_context_attr_ids { UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, @@ -169,9 +177,16 @@ enum uverbs_attrs_destroy_qp_cmd_attr_ids { UVERBS_ATTR_DESTROY_QP_RESP, }; +enum uverbs_attrs_qp_attach_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, +}; + enum uverbs_methods_qp { UVERBS_METHOD_QP_CREATE, UVERBS_METHOD_QP_DESTROY, + UVERBS_METHOD_QP_ATTACH_COMP_CNTR, }; enum uverbs_attrs_create_srq_cmd_attr_ids { @@ -438,4 +453,32 @@ enum uverbs_attrs_query_gid_entry_cmd_attr_ids { UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, }; +enum uverbs_methods_comp_cntr { + UVERBS_METHOD_COMP_CNTR_CREATE, + UVERBS_METHOD_COMP_CNTR_DESTROY, + UVERBS_METHOD_COMP_CNTR_MODIFY, + UVERBS_METHOD_COMP_CNTR_READ, +}; + +enum uverbs_attrs_create_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, +}; + +enum uverbs_attrs_destroy_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, +}; + +enum uverbs_attrs_modify_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, + UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, + UVERBS_ATTR_MODIFY_COMP_CNTR_OP, + UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, +}; + +enum uverbs_attrs_read_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_READ_COMP_CNTR_HANDLE, + UVERBS_ATTR_READ_COMP_CNTR_ENTRY, + UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, +}; + #endif diff --git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h index 51030c27d..21f86cc7b 100644 --- a/kernel-headers/rdma/ib_user_ioctl_verbs.h +++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h @@ -300,4 +300,23 @@ struct ib_uverbs_buffer_desc { __aligned_u64 length; }; +enum ib_uverbs_comp_cntr_entry { + IB_UVERBS_COMP_CNTR_ENTRY_COMP, + IB_UVERBS_COMP_CNTR_ENTRY_ERR, +}; + +enum ib_uverbs_comp_cntr_modify_op { + IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, + IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, +}; + +enum ib_uverbs_qp_attach_comp_cntr_op { + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND = 1 << 0, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV = 1 << 1, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ = 1 << 2, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ = 1 << 3, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE = 1 << 4, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE = 1 << 5, +}; + #endif diff --git a/kernel-headers/rdma/ib_user_verbs.h b/kernel-headers/rdma/ib_user_verbs.h index d2aeadb6d..565301594 100644 --- a/kernel-headers/rdma/ib_user_verbs.h +++ b/kernel-headers/rdma/ib_user_verbs.h @@ -1379,4 +1379,37 @@ enum ib_uverbs_raw_packet_caps { IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP = 1 << 3, }; +/* + * struct ib_uverbs_clock_info - timecounter state shared with userspace + * + * Drivers that use a software timecounter over a free-running hardware + * cycle counter can map this page read-only into userspace, allowing + * conversion of hardware timestamps to system time without a syscall. + * + * Synchronization uses a sequence counter (@sign): the kernel sets bit 0 + * before updating, then advances by 2 after. Userspace must retry the read + * if @sign is odd or changed during the read. + * + * @sign: Sequence counter (bit 0 = update in progress) + * @resv: Reserved + * @nsec: Nanoseconds at last update + * @cycles: Cycle counter value at last update + * @frac: Fractional nanoseconds at last update + * @mult: Cycle-to-nanosecond multiplier + * @shift: Cycle-to-nanosecond shift + * @mask: Cycle counter bitmask + * @overflow_period: Max interval (nsec) between reads before counter wraps + */ +struct ib_uverbs_clock_info { + __u32 sign; + __u32 resv; + __aligned_u64 nsec; + __aligned_u64 cycles; + __aligned_u64 frac; + __u32 mult; + __u32 shift; + __aligned_u64 mask; + __aligned_u64 overflow_period; +}; + #endif /* IB_USER_VERBS_H */ diff --git a/kernel-headers/rdma/ionic-abi.h b/kernel-headers/rdma/ionic-abi.h index 7b589d3e9..2c70ac149 100644 --- a/kernel-headers/rdma/ionic-abi.h +++ b/kernel-headers/rdma/ionic-abi.h @@ -48,6 +48,7 @@ struct ionic_ctx_resp { __u8 expdb_qtypes; __u8 rsvd2[3]; + __aligned_u64 phc_offset; }; struct ionic_qdesc { diff --git a/kernel-headers/rdma/irdma-abi.h b/kernel-headers/rdma/irdma-abi.h index 36f20802b..38155affc 100644 --- a/kernel-headers/rdma/irdma-abi.h +++ b/kernel-headers/rdma/irdma-abi.h @@ -88,6 +88,7 @@ struct irdma_create_srq_resp { struct irdma_create_qp_req { __aligned_u64 user_wqe_bufs; __aligned_u64 user_compl_ctx; + __aligned_u64 legacy_dontuse[2]; }; struct irdma_mem_reg_req { diff --git a/kernel-headers/rdma/mlx5-abi.h b/kernel-headers/rdma/mlx5-abi.h index 8a6ad6c68..a39226cd6 100644 --- a/kernel-headers/rdma/mlx5-abi.h +++ b/kernel-headers/rdma/mlx5-abi.h @@ -472,17 +472,10 @@ struct mlx5_ib_modify_wq { __u32 reserved; }; -struct mlx5_ib_clock_info { - __u32 sign; - __u32 resv; - __aligned_u64 nsec; - __aligned_u64 cycles; - __aligned_u64 frac; - __u32 mult; - __u32 shift; - __aligned_u64 mask; - __aligned_u64 overflow_period; -}; +/* + * deprecated, see struct ib_uverbs_clock_info from ib_user_verbs.h + */ +#define mlx5_ib_clock_info ib_uverbs_clock_info enum mlx5_ib_mmap_cmd { MLX5_IB_MMAP_REGULAR_PAGE = 0, diff --git a/kernel-headers/rdma/rdma_netlink.h b/kernel-headers/rdma/rdma_netlink.h index aac9782dd..3af946ecb 100644 --- a/kernel-headers/rdma/rdma_netlink.h +++ b/kernel-headers/rdma/rdma_netlink.h @@ -604,6 +604,11 @@ enum rdma_nldev_attr { RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES, /* u32 */ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY, /* u64 */ + /* + * Resource summary entry maximum value. + */ + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, /* u64 */ + /* * Always the end */ diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt index eaf25a1ae..6fd823e86 100644 --- a/libibverbs/CMakeLists.txt +++ b/libibverbs/CMakeLists.txt @@ -27,6 +27,7 @@ rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map" cmd.c cmd_ah.c cmd_counters.c + cmd_comp_cntr.c cmd_cq.c cmd_device.c cmd_dm.c diff --git a/libibverbs/cmd_comp_cntr.c b/libibverbs/cmd_comp_cntr.c new file mode 100644 index 000000000..cc989f5f6 --- /dev/null +++ b/libibverbs/cmd_comp_cntr.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB +/* + * Copyright Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#include + +int ibv_cmd_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr *comp_cntr, + struct ibv_command_buffer *link) +{ + DECLARE_COMMAND_BUFFER_LINK(cmdb, UVERBS_OBJECT_COMP_CNTR, + UVERBS_METHOD_COMP_CNTR_CREATE, 1, link); + struct ib_uverbs_attr *handle; + int ret; + + comp_cntr->context = context; + + handle = fill_attr_out_obj(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE); + + ret = execute_ioctl(context, cmdb); + if (ret) + return errno; + + comp_cntr->handle = read_attr_obj(UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, handle); + return 0; +} + +int ibv_cmd_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_DESTROY, 1); + int ret; + + fill_attr_in_obj(cmdb, UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, comp_cntr->handle); + + ret = execute_ioctl(comp_cntr->context, cmdb); + if (verbs_is_destroy_err(&ret)) + return ret; + + return 0; +} + +static int ibv_icmd_modify_comp_cntr(struct ibv_comp_cntr *comp_cntr, + uint8_t entry, uint8_t op, uint64_t value) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_MODIFY, 4); + + fill_attr_in_obj(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, comp_cntr->handle); + fill_attr_const_in(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, entry); + fill_attr_const_in(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_OP, op); + fill_attr_in_uint64(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, value); + + return execute_ioctl(comp_cntr->context, cmdb); +} + +int ibv_cmd_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value) +{ + return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP, + IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, value); +} + +int ibv_cmd_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value) +{ + return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR, + IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, value); +} + +int ibv_cmd_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount) +{ + return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP, + IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, amount); +} + +int ibv_cmd_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount) +{ + return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR, + IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, amount); +} + +static int ibv_icmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, + uint8_t entry, uint64_t *value) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_READ, 3); + + fill_attr_in_obj(cmdb, UVERBS_ATTR_READ_COMP_CNTR_HANDLE, comp_cntr->handle); + fill_attr_const_in(cmdb, UVERBS_ATTR_READ_COMP_CNTR_ENTRY, entry); + fill_attr_out_ptr(cmdb, UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, value); + + return execute_ioctl(comp_cntr->context, cmdb); +} + +int ibv_cmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value) +{ + return ibv_icmd_read_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP, value); +} + +int ibv_cmd_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value) +{ + return ibv_icmd_read_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR, value); +} + +int ibv_cmd_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_QP, UVERBS_METHOD_QP_ATTACH_COMP_CNTR, 3); + uint32_t op_mask = 0; + + fill_attr_in_obj(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, qp->handle); + fill_attr_in_obj(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, comp_cntr->handle); + + if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_SEND) + op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND; + if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RECV) + op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV; + if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) + op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ; + if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ) + op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ; + if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) + op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE; + if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE) + op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE; + + fill_attr_in_uint32(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, op_mask); + + return execute_ioctl(qp->context, cmdb); +} diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c index ba22372a2..a7b4072b7 100644 --- a/libibverbs/cmd_device.c +++ b/libibverbs/cmd_device.c @@ -698,3 +698,43 @@ int ibv_cmd_query_device_any(struct ibv_context *context, return 0; } + +int ibv_cmd_query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps, + size_t caps_size) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_DEVICE, + UVERBS_METHOD_QUERY_COMP_CNTR_CAPS, 3); + uint32_t supported_qp_attach_ops; + int ret; + + if (caps_size < sizeof(struct ibv_comp_cntr_caps)) + return EINVAL; + + fill_attr_out_ptr(cmdb, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + &caps->max_counters); + fill_attr_out_ptr(cmdb, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + &caps->max_value); + fill_attr_out_ptr(cmdb, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, + &supported_qp_attach_ops); + + ret = execute_ioctl(context, cmdb); + if (ret) + return ret; + + caps->supported_qp_attach_ops = 0; + if (supported_qp_attach_ops & IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND) + caps->supported_qp_attach_ops |= IBV_QP_ATTACH_COMP_CNTR_OP_SEND; + if (supported_qp_attach_ops & IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV) + caps->supported_qp_attach_ops |= IBV_QP_ATTACH_COMP_CNTR_OP_RECV; + if (supported_qp_attach_ops & IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) + caps->supported_qp_attach_ops |= IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ; + if (supported_qp_attach_ops & IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ) + caps->supported_qp_attach_ops |= IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ; + if (supported_qp_attach_ops & IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) + caps->supported_qp_attach_ops |= IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE; + if (supported_qp_attach_ops & IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE) + caps->supported_qp_attach_ops |= IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE; + + return 0; +} diff --git a/libibverbs/driver.h b/libibverbs/driver.h index ec72c7748..1f4ca3743 100644 --- a/libibverbs/driver.h +++ b/libibverbs/driver.h @@ -391,6 +391,8 @@ struct verbs_context_ops { void (*cq_event)(struct ibv_cq *cq); struct ibv_ah *(*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr); + struct ibv_comp_cntr *(*create_comp_cntr)(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *attr); struct ibv_counters *(*create_counters)(struct ibv_context *context, struct ibv_counters_init_attr *init_attr); struct ibv_cq *(*create_cq)(struct ibv_context *context, int cqe, @@ -424,6 +426,7 @@ struct verbs_context_ops { int (*dealloc_td)(struct ibv_td *td); int (*dereg_mr)(struct verbs_mr *vmr); int (*destroy_ah)(struct ibv_ah *ah); + int (*destroy_comp_cntr)(struct ibv_comp_cntr *comp_cntr); int (*destroy_counters)(struct ibv_counters *counters); int (*destroy_cq)(struct ibv_cq *cq); int (*destroy_flow)(struct ibv_flow *flow); @@ -445,6 +448,9 @@ struct verbs_context_ops { uint32_t mr_handle); struct ibv_pd *(*import_pd)(struct ibv_context *context, uint32_t pd_handle); + int (*inc_comp_cntr)(struct ibv_comp_cntr *comp_cntr, uint64_t amount); + int (*inc_err_comp_cntr)(struct ibv_comp_cntr *comp_cntr, + uint64_t amount); int (*modify_cq)(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr); int (*modify_flow_action_esp)(struct ibv_flow_action *action, struct ibv_flow_action_esp_attr *attr); @@ -469,6 +475,12 @@ struct verbs_context_ops { struct ibv_ops_wr **bad_op); int (*post_srq_recv)(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, struct ibv_recv_wr **bad_recv_wr); + int (*qp_attach_comp_cntr)(struct ibv_qp *qp, + struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr); + int (*query_comp_cntr_caps)(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps, + size_t caps_size); int (*query_device_ex)(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, @@ -485,10 +497,13 @@ struct verbs_context_ops { int (*query_rt_values)(struct ibv_context *context, struct ibv_values_ex *values); int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); + int (*read_comp_cntr)(struct ibv_comp_cntr *comp_cntr, uint64_t *value); int (*read_counters)(struct ibv_counters *counters, uint64_t *counters_value, uint32_t ncounters, uint32_t flags); + int (*read_err_comp_cntr)(struct ibv_comp_cntr *comp_cntr, + uint64_t *value); struct ibv_mr *(*reg_dm_mr)(struct ibv_pd *pd, struct ibv_dm *dm, uint64_t dm_offset, size_t length, unsigned int access); @@ -503,7 +518,10 @@ struct verbs_context_ops { int (*rereg_mr)(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, void *addr, size_t length, int access); int (*resize_cq)(struct ibv_cq *cq, int cqe); + int (*set_comp_cntr)(struct ibv_comp_cntr *comp_cntr, uint64_t value); int (*set_ece)(struct ibv_qp *qp, struct ibv_ece *ece); + int (*set_err_comp_cntr)(struct ibv_comp_cntr *comp_cntr, + uint64_t value); void (*unimport_dm)(struct ibv_dm *dm); void (*unimport_mr)(struct ibv_mr *mr); void (*unimport_pd)(struct ibv_pd *pd); @@ -777,6 +795,24 @@ int ibv_cmd_export_dmabuf_fd(struct ibv_context *ctx, off_t pg_off); int ibv_cmd_alloc_dmah(struct ibv_context *ctx, struct verbs_dmah *st, struct ibv_dmah_init_attr *attr); int ibv_cmd_free_dmah(struct verbs_dmah *dmah); + +int ibv_cmd_query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps, + size_t caps_size); +int ibv_cmd_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr *comp_cntr, + struct ibv_command_buffer *link); +int ibv_cmd_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr); +int ibv_cmd_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount); +int ibv_cmd_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount); +int ibv_cmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value); +int ibv_cmd_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value); +int ibv_cmd_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value); +int ibv_cmd_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value); +int ibv_cmd_qp_attach_comp_cntr(struct ibv_qp *qp, + struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr); + int ibv_cmd_reg_dm_mr(struct ibv_pd *pd, struct verbs_dm *dm, uint64_t offset, size_t length, unsigned int access, struct verbs_mr *vmr, diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c index a27e22905..3eacff3ab 100644 --- a/libibverbs/dummy_ops.c +++ b/libibverbs/dummy_ops.c @@ -161,6 +161,13 @@ static struct ibv_ah *create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) return NULL; } +static struct ibv_comp_cntr *create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *attr) +{ + errno = EOPNOTSUPP; + return NULL; +} + static struct ibv_counters *create_counters(struct ibv_context *context, struct ibv_counters_init_attr *init_attr) { @@ -271,6 +278,11 @@ static int destroy_ah(struct ibv_ah *ah) return EOPNOTSUPP; } +static int destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr) +{ + return EOPNOTSUPP; +} + static int destroy_counters(struct ibv_counters *counters) { return EOPNOTSUPP; @@ -359,6 +371,16 @@ static struct ibv_pd *import_pd(struct ibv_context *context, return NULL; } +static int inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount) +{ + return EOPNOTSUPP; +} + +static int inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount) +{ + return EOPNOTSUPP; +} + static int modify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr) { return EOPNOTSUPP; @@ -435,6 +457,20 @@ static int post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, return EOPNOTSUPP; } +static int qp_attach_comp_cntr(struct ibv_qp *qp, + struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr) +{ + return EOPNOTSUPP; +} + +static int query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps, + size_t caps_size) +{ + return EOPNOTSUPP; +} + static int query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size) @@ -482,6 +518,11 @@ static int query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr) return EOPNOTSUPP; } +static int read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value) +{ + return EOPNOTSUPP; +} + static int read_counters(struct ibv_counters *counters, uint64_t *counters_value, uint32_t ncounters, @@ -490,6 +531,11 @@ static int read_counters(struct ibv_counters *counters, return EOPNOTSUPP; } +static int read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value) +{ + return EOPNOTSUPP; +} + static struct ibv_mr *reg_dm_mr(struct ibv_pd *pd, struct ibv_dm *dm, uint64_t dm_offset, size_t length, unsigned int access) @@ -578,11 +624,21 @@ static int resize_cq(struct ibv_cq *cq, int cqe) return EOPNOTSUPP; } +static int set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value) +{ + return EOPNOTSUPP; +} + static int set_ece(struct ibv_qp *qp, struct ibv_ece *ece) { return EOPNOTSUPP; } +static int set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value) +{ + return EOPNOTSUPP; +} + static void unimport_dm(struct ibv_dm *dm) { } @@ -623,6 +679,7 @@ const struct verbs_context_ops verbs_dummy_ops = { close_xrcd, cq_event, create_ah, + create_comp_cntr, create_counters, create_cq, create_cq_ex, @@ -640,6 +697,7 @@ const struct verbs_context_ops verbs_dummy_ops = { dealloc_td, dereg_mr, destroy_ah, + destroy_comp_cntr, destroy_counters, destroy_cq, destroy_flow, @@ -657,6 +715,8 @@ const struct verbs_context_ops verbs_dummy_ops = { import_dm, import_mr, import_pd, + inc_comp_cntr, + inc_err_comp_cntr, modify_cq, modify_flow_action_esp, modify_qp, @@ -670,6 +730,8 @@ const struct verbs_context_ops verbs_dummy_ops = { post_send, post_srq_ops, post_srq_recv, + qp_attach_comp_cntr, + query_comp_cntr_caps, query_device_ex, query_ece, query_port, @@ -678,7 +740,9 @@ const struct verbs_context_ops verbs_dummy_ops = { query_qp_data_in_order, query_rt_values, query_srq, + read_comp_cntr, read_counters, + read_err_comp_cntr, reg_dm_mr, reg_dmabuf_mr, reg_mr, @@ -686,7 +750,9 @@ const struct verbs_context_ops verbs_dummy_ops = { req_notify_cq, rereg_mr, resize_cq, + set_comp_cntr, set_ece, + set_err_comp_cntr, unimport_dm, unimport_mr, unimport_pd, @@ -758,6 +824,7 @@ void verbs_set_ops(struct verbs_context *vctx, SET_PRIV_OP(ctx, create_ah); SET_PRIV_OP(ctx, create_cq); SET_PRIV_OP_IC(vctx, create_cq_ex); + SET_PRIV_OP_IC(vctx, create_comp_cntr); SET_OP2(vctx, ibv_create_flow, create_flow); SET_OP(vctx, create_flow_action_esp); SET_PRIV_OP(ctx, create_qp); @@ -773,6 +840,7 @@ void verbs_set_ops(struct verbs_context *vctx, SET_OP(vctx, destroy_counters); SET_PRIV_OP(ctx, dereg_mr); SET_PRIV_OP(ctx, destroy_ah); + SET_PRIV_OP_IC(vctx, destroy_comp_cntr); SET_PRIV_OP(ctx, destroy_cq); SET_OP2(vctx, ibv_destroy_flow, destroy_flow); SET_OP(vctx, destroy_flow_action); @@ -789,6 +857,8 @@ void verbs_set_ops(struct verbs_context *vctx, SET_PRIV_OP_IC(vctx, import_dm); SET_PRIV_OP_IC(vctx, import_mr); SET_PRIV_OP_IC(vctx, import_pd); + SET_PRIV_OP_IC(vctx, inc_comp_cntr); + SET_PRIV_OP_IC(vctx, inc_err_comp_cntr); SET_OP(vctx, modify_cq); SET_OP(vctx, modify_flow_action_esp); SET_PRIV_OP(ctx, modify_qp); @@ -802,6 +872,8 @@ void verbs_set_ops(struct verbs_context *vctx, SET_OP(ctx, post_send); SET_OP(vctx, post_srq_ops); SET_OP(ctx, post_srq_recv); + SET_PRIV_OP_IC(vctx, qp_attach_comp_cntr); + SET_PRIV_OP_IC(ctx, query_comp_cntr_caps); SET_OP(vctx, query_device_ex); SET_PRIV_OP_IC(vctx, query_ece); SET_PRIV_OP_IC(ctx, query_port); @@ -809,7 +881,9 @@ void verbs_set_ops(struct verbs_context *vctx, SET_PRIV_OP(ctx, query_qp); SET_PRIV_OP_IC(ctx, query_qp_data_in_order); SET_OP(vctx, query_rt_values); + SET_PRIV_OP_IC(vctx, read_comp_cntr); SET_OP(vctx, read_counters); + SET_PRIV_OP_IC(vctx, read_err_comp_cntr); SET_PRIV_OP(ctx, query_srq); SET_OP(vctx, reg_dm_mr); SET_PRIV_OP_IC(vctx, reg_dmabuf_mr); @@ -818,7 +892,9 @@ void verbs_set_ops(struct verbs_context *vctx, SET_OP(ctx, req_notify_cq); SET_PRIV_OP(ctx, rereg_mr); SET_PRIV_OP(ctx, resize_cq); + SET_PRIV_OP_IC(vctx, set_comp_cntr); SET_PRIV_OP_IC(vctx, set_ece); + SET_PRIV_OP_IC(vctx, set_err_comp_cntr); SET_PRIV_OP_IC(vctx, unimport_dm); SET_PRIV_OP_IC(vctx, unimport_mr); SET_PRIV_OP_IC(vctx, unimport_pd); diff --git a/libibverbs/examples/devinfo.c b/libibverbs/examples/devinfo.c index 1892ccd50..72022842e 100644 --- a/libibverbs/examples/devinfo.c +++ b/libibverbs/examples/devinfo.c @@ -507,10 +507,28 @@ static void print_raw_packet_caps(uint32_t raw_packet_caps) printf("\t\t\t\t\tDelay drop\n"); } +static void print_comp_cntr_attach_ops(uint32_t ops) +{ + printf("\tcomp_cntr_caps:\n"); + if (ops & IBV_QP_ATTACH_COMP_CNTR_OP_SEND) + printf("\t\t\t\t\tQP_ATTACH_SEND\n"); + if (ops & IBV_QP_ATTACH_COMP_CNTR_OP_RECV) + printf("\t\t\t\t\tQP_ATTACH_RECV\n"); + if (ops & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) + printf("\t\t\t\t\tQP_ATTACH_RDMA_READ\n"); + if (ops & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ) + printf("\t\t\t\t\tQP_ATTACH_REMOTE_RDMA_READ\n"); + if (ops & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) + printf("\t\t\t\t\tQP_ATTACH_RDMA_WRITE\n"); + if (ops & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE) + printf("\t\t\t\t\tQP_ATTACH_REMOTE_RDMA_WRITE\n"); +} + static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) { struct ibv_context *ctx; struct ibv_device_attr_ex device_attr = {}; + struct ibv_comp_cntr_caps cc_caps = {}; struct ibv_port_attr port_attr; uint64_t port_speed; int rc = 0; @@ -592,6 +610,10 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) printf("\tmax_srq_sge:\t\t\t%d\n", device_attr.orig_attr.max_srq_sge); } printf("\tmax_pkeys:\t\t\t%d\n", device_attr.orig_attr.max_pkeys); + if (!ibv_query_comp_cntr_caps(ctx, &cc_caps)) { + printf("\tmax_comp_cntr:\t\t\t%d\n", cc_caps.max_counters); + print_comp_cntr_attach_ops(cc_caps.supported_qp_attach_ops); + } printf("\tlocal_ca_ack_delay:\t\t%d\n", device_attr.orig_attr.local_ca_ack_delay); print_odp_caps(&device_attr); diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in index 7329bce5a..e7a1598cd 100644 --- a/libibverbs/libibverbs.map.in +++ b/libibverbs/libibverbs.map.in @@ -186,7 +186,17 @@ IBVERBS_1.17 { IBVERBS_1.18 { global: + _ibv_query_comp_cntr_caps; + ibv_create_comp_cntr; + ibv_destroy_comp_cntr; ibv_export_buf_dmabuf_fd; + ibv_inc_comp_cntr; + ibv_inc_err_comp_cntr; + ibv_qp_attach_comp_cntr; + ibv_read_comp_cntr; + ibv_read_err_comp_cntr; + ibv_set_comp_cntr; + ibv_set_err_comp_cntr; } IBVERBS_1.17; /* If any symbols in this stanza change ABI then the entire staza gets a new symbol @@ -207,6 +217,7 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_attach_mcast; ibv_cmd_close_xrcd; ibv_cmd_create_ah; + ibv_cmd_create_comp_cntr; ibv_cmd_create_counters; ibv_cmd_create_cq; ibv_cmd_create_cq_ex; @@ -224,6 +235,7 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_dealloc_pd; ibv_cmd_dereg_mr; ibv_cmd_destroy_ah; + ibv_cmd_destroy_comp_cntr; ibv_cmd_destroy_counters; ibv_cmd_destroy_cq; ibv_cmd_destroy_flow; @@ -237,6 +249,8 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_free_dm; ibv_cmd_free_dmah; ibv_cmd_get_context; + ibv_cmd_inc_comp_cntr; + ibv_cmd_inc_err_comp_cntr; ibv_cmd_modify_cq; ibv_cmd_modify_flow_action_esp; ibv_cmd_modify_qp; @@ -249,6 +263,8 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_post_recv; ibv_cmd_post_send; ibv_cmd_post_srq_recv; + ibv_cmd_qp_attach_comp_cntr; + ibv_cmd_query_comp_cntr_caps; ibv_cmd_query_context; ibv_cmd_query_device_any; ibv_cmd_query_mr; @@ -256,7 +272,9 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_query_port_speed; ibv_cmd_query_qp; ibv_cmd_query_srq; + ibv_cmd_read_comp_cntr; ibv_cmd_read_counters; + ibv_cmd_read_err_comp_cntr; ibv_cmd_reg_dm_mr; ibv_cmd_reg_dmabuf_mr; ibv_cmd_reg_mr; @@ -264,6 +282,8 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_req_notify_cq; ibv_cmd_rereg_mr; ibv_cmd_resize_cq; + ibv_cmd_set_comp_cntr; + ibv_cmd_set_err_comp_cntr; ibv_dmabuf_heap_alloc; ibv_dmabuf_heap_cc_shared_init; ibv_dmabuf_heap_destroy; diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt index f8eca8668..0efe0b04b 100644 --- a/libibverbs/man/CMakeLists.txt +++ b/libibverbs/man/CMakeLists.txt @@ -15,6 +15,7 @@ rdma_man_pages( ibv_create_ah.3 ibv_create_ah_from_wc.3 ibv_create_comp_channel.3 + ibv_create_comp_cntr.3.md ibv_create_counters.3.md ibv_create_cq.3 ibv_create_cq_ex.3 @@ -58,6 +59,8 @@ rdma_man_pages( ibv_post_send.3 ibv_post_srq_ops.3 ibv_post_srq_recv.3 + ibv_qp_attach_comp_cntr.3.md + ibv_query_comp_cntr_caps.3.md ibv_query_device.3 ibv_query_device_ex.3 ibv_query_ece.3.md @@ -102,6 +105,13 @@ rdma_alias_man_pages( ibv_create_ah.3 ibv_destroy_ah.3 ibv_create_ah_from_wc.3 ibv_init_ah_from_wc.3 ibv_create_comp_channel.3 ibv_destroy_comp_channel.3 + ibv_create_comp_cntr.3 ibv_destroy_comp_cntr.3 + ibv_create_comp_cntr.3 ibv_set_comp_cntr.3 + ibv_create_comp_cntr.3 ibv_set_err_comp_cntr.3 + ibv_create_comp_cntr.3 ibv_inc_comp_cntr.3 + ibv_create_comp_cntr.3 ibv_inc_err_comp_cntr.3 + ibv_create_comp_cntr.3 ibv_read_comp_cntr.3 + ibv_create_comp_cntr.3 ibv_read_err_comp_cntr.3 ibv_create_counters.3 ibv_destroy_counters.3 ibv_create_cq.3 ibv_destroy_cq.3 ibv_create_flow.3 ibv_destroy_flow.3 diff --git a/libibverbs/man/ibv_create_comp_cntr.3.md b/libibverbs/man/ibv_create_comp_cntr.3.md new file mode 100644 index 000000000..aa5ce4321 --- /dev/null +++ b/libibverbs/man/ibv_create_comp_cntr.3.md @@ -0,0 +1,180 @@ +--- +date: 2026-02-09 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: ibv_create_comp_cntr +tagline: Verbs +--- + +# NAME + +**ibv_create_comp_cntr**, **ibv_destroy_comp_cntr** - Create or destroy a +completion counter + +**ibv_set_comp_cntr**, **ibv_set_err_comp_cntr** - Set the value of a +completion or error counter + +**ibv_inc_comp_cntr**, **ibv_inc_err_comp_cntr** - Increment a completion or +error counter + +**ibv_read_comp_cntr**, **ibv_read_err_comp_cntr** - Read the value of a +completion or error counter + +# SYNOPSIS + +```c +#include + +struct ibv_comp_cntr *ibv_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *cc_attr); + +int ibv_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr); + +int ibv_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value); +int ibv_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value); +int ibv_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount); +int ibv_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount); +int ibv_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value); +int ibv_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value); +``` + +# DESCRIPTION + +Completion counters provide a lightweight completion mechanism as an +alternative or extension to completion queues (CQs). Rather than generating +individual completion queue entries, a completion counter tracks the aggregate +number of completed operations. This makes them well suited for applications +that need to know how many requests have completed without requiring +per-request details, such as credit based flow control or tracking responses +from remote peers. + +Each completion counter maintains two distinct 64-bit values: a completion +count that is incremented on successful completions, and an error count that +is incremented when operations complete in error. + +**ibv_create_comp_cntr**() allocates a new completion counter for the RDMA +device context *context*. The properties of the counter are defined by +*cc_attr*. The maximum number of completion counters a device supports is +reported by **ibv_query_comp_cntr_caps**(3). + +**ibv_destroy_comp_cntr**() releases all resources associated with the +completion counter *comp_cntr*. The counter must not be attached to any QP +when destroyed. + +**ibv_set_comp_cntr**() sets the completion count of *comp_cntr* to *value*. + +**ibv_set_err_comp_cntr**() sets the error count of *comp_cntr* to *value*. + +**ibv_inc_comp_cntr**() increments the completion count of *comp_cntr* by +*amount*. + +**ibv_inc_err_comp_cntr**() increments the error count of *comp_cntr* by +*amount*. + +**ibv_read_comp_cntr**() reads the current completion count of *comp_cntr* +into *value*. + +**ibv_read_err_comp_cntr**() reads the current error count of *comp_cntr* +into *value*. + +# ARGUMENTS + +## ibv_comp_cntr + +```c +struct ibv_comp_cntr { + struct ibv_context *context; + uint32_t handle; +}; +``` + +*context* +: Device context associated with the completion counter. + +*handle* +: Kernel object handle for the completion counter. + +## ibv_comp_cntr_init_attr + +```c +enum ibv_comp_cntr_type { + IBV_COMP_CNTR_TYPE_WRS, + IBV_COMP_CNTR_TYPE_BYTES, +}; + +struct ibv_comp_cntr_init_attr { + uint32_t comp_mask; + enum ibv_comp_cntr_type type; + uint32_t flags; +}; +``` + +*comp_mask* +: Bitmask specifying what fields in the structure are valid. + +*type* +: The counting mode for the completion counter. Not all devices support + all modes. + **IBV_COMP_CNTR_TYPE_WRS** counts completed work requests (default). + **IBV_COMP_CNTR_TYPE_BYTES** counts completed bytes. + +*flags* +: Reserved for future use, for now must be 0. + +# RETURN VALUE + +**ibv_create_comp_cntr**() returns a pointer to the allocated ibv_comp_cntr +object, or NULL if the request fails (and sets errno to indicate the failure +reason). + +**ibv_destroy_comp_cntr**(), **ibv_set_comp_cntr**(), +**ibv_set_err_comp_cntr**(), **ibv_inc_comp_cntr**(), +**ibv_inc_err_comp_cntr**(), **ibv_read_comp_cntr**(), and +**ibv_read_err_comp_cntr**() return 0 on success, or the value of errno on +failure (which indicates the failure reason). + +# ERRORS + +ENOTSUP +: Completion counters are not supported on this device, or the + requested operation is not supported for the given counter + configuration. + +ENOMEM +: Not enough resources to create the completion counter. + +EINVAL +: Invalid argument(s) passed. + +EBUSY +: The completion counter is still attached to a QP + (**ibv_destroy_comp_cntr**() only). + +# NOTES + +Counter values must only be updated using **ibv_set_comp_cntr**(), +**ibv_set_err_comp_cntr**(), **ibv_inc_comp_cntr**(), or +**ibv_inc_err_comp_cntr**(). + +Updates made to counter values (e.g. via **ibv_set_comp_cntr**() or +**ibv_inc_comp_cntr**()) may not be immediately visible when reading the +counter via **ibv_read_comp_cntr**() or **ibv_read_err_comp_cntr**(). A small +delay may occur between the update and the observed value. However, the final +updated value will eventually be reflected. + +Applications should ensure that the counter value is stable before calling +**ibv_set_comp_cntr**() or **ibv_set_err_comp_cntr**(). Otherwise, concurrent +updates may be lost. + +# SEE ALSO + +**ibv_query_comp_cntr_caps**(3), **ibv_qp_attach_comp_cntr**(3), +**efadv_create_comp_cntr**(3), **ibv_create_cq**(3), **ibv_create_cq_ex**(3), +**ibv_create_qp**(3) + +# AUTHORS + +Michael Margolin diff --git a/libibverbs/man/ibv_qp_attach_comp_cntr.3.md b/libibverbs/man/ibv_qp_attach_comp_cntr.3.md new file mode 100644 index 000000000..e7f2eef54 --- /dev/null +++ b/libibverbs/man/ibv_qp_attach_comp_cntr.3.md @@ -0,0 +1,123 @@ +--- +date: 2026-02-09 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: ibv_qp_attach_comp_cntr +tagline: Verbs +--- + +# NAME + +**ibv_qp_attach_comp_cntr** - Attach a completion counter to a QP + +# SYNOPSIS + +```c +#include + +int ibv_qp_attach_comp_cntr(struct ibv_qp *qp, + struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr); +``` + +# DESCRIPTION + +**ibv_qp_attach_comp_cntr**() attaches the completion counter *comp_cntr* to +the queue pair *qp*. The *attr* argument specifies which operation types +should update the counter. + +The QP must be in **IBV_QPS_RESET** or **IBV_QPS_INIT** state when attaching +a completion counter. Attempting to attach a counter to a QP in any other +state will fail with EINVAL. + +The completion counter starts collecting values for the specified QP once +attached. Attaching the same completion counter to multiple QPs will +accumulate values from all attached QPs into the same counter. + +The *op_mask* field controls which operation completions are counted. Local +operations (**IBV_QP_ATTACH_COMP_CNTR_OP_SEND**, **IBV_QP_ATTACH_COMP_CNTR_OP_RECV**, +**IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ**, **IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE**) +count completions initiated by the local QP. Remote operations +(**IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ**, +**IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE**) count completions of incoming +RDMA operations initiated by the remote side. Supported *op_mask* values may +vary by device and can be queried using **ibv_query_comp_cntr_caps**(3); +unsupported values will result in an ENOTSUP error. + +Multiple completion counters can be attached to the same QP, provided their +*op_mask* values do not overlap. Each QP and operation type pair can be +associated with at most one completion counter. Attempting to attach a +counter with an *op_mask* that conflicts with an already attached counter +will fail. + +There is no explicit detach operation. A completion counter is implicitly +detached when the QP it is attached to is destroyed. A completion counter +cannot be destroyed while it is still attached to any QP; the QP must be +destroyed first. + +# ARGUMENTS + +*qp* +: The queue pair to attach the completion counter to. + +*comp_cntr* +: The completion counter to attach, previously created with + **ibv_create_comp_cntr**(). + +*attr* +: Attach attributes specifying which operation types update the counter. + +## ibv_qp_attach_comp_cntr_attr + +```c +enum ibv_qp_attach_comp_cntr_op { + IBV_QP_ATTACH_COMP_CNTR_OP_SEND = 1 << 0, + IBV_QP_ATTACH_COMP_CNTR_OP_RECV = 1 << 1, + IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ = 1 << 2, + IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ = 1 << 3, + IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE = 1 << 4, + IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE = 1 << 5, +}; + +struct ibv_qp_attach_comp_cntr_attr { + uint32_t comp_mask; + uint32_t op_mask; +}; +``` + +*comp_mask* +: Bitmask specifying what fields in the structure are valid. Used + for extensibility, should be set to 0. + +*op_mask* +: Bitmask of **ibv_qp_attach_comp_cntr_op** values specifying which + operation types should update the counter. + +# RETURN VALUE + +**ibv_qp_attach_comp_cntr**() returns 0 on success, or the value of errno on +failure (which indicates the failure reason). + +# ERRORS + +EINVAL +: Invalid argument(s) passed. + +ENOTSUP +: Requested operation is not supported on this device. + +EBUSY +: The *op_mask* overlaps with a completion counter already attached + to this QP. + +# SEE ALSO + +**ibv_create_comp_cntr**(3), **ibv_query_comp_cntr_caps**(3), +**ibv_create_qp**(3) + +# AUTHORS + +Michael Margolin diff --git a/libibverbs/man/ibv_query_comp_cntr_caps.3.md b/libibverbs/man/ibv_query_comp_cntr_caps.3.md new file mode 100644 index 000000000..c7a367d8c --- /dev/null +++ b/libibverbs/man/ibv_query_comp_cntr_caps.3.md @@ -0,0 +1,71 @@ +--- +date: 2026-07-17 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: ibv_query_comp_cntr_caps +tagline: Verbs +--- + +# NAME + +**ibv_query_comp_cntr_caps** - Query completion counter capabilities + +# SYNOPSIS + +```c +#include + +int ibv_query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps); +``` + +# DESCRIPTION + +**ibv_query_comp_cntr_caps**() queries the completion counter capabilities +of the RDMA device associated with *context*. + +# ARGUMENTS + +## ibv_comp_cntr_caps + +```c +struct ibv_comp_cntr_caps { + uint64_t max_value; + uint32_t max_counters; + uint32_t supported_qp_attach_ops; +}; +``` + +*max_value* +: The maximum value a completion counter can hold. A subsequent + increment that would exceed this value wraps the counter to zero. + +*max_counters* +: The maximum number of completion counters that can be created on + this device. + +*supported_qp_attach_ops* +: A bitmask of **ibv_qp_attach_comp_cntr_op** values indicating which + attach operations are supported by the device. See + **ibv_qp_attach_comp_cntr**(3) for the list of operations. + +# RETURN VALUE + +**ibv_query_comp_cntr_caps**() returns 0 on success, or the value of errno +on failure (which indicates the failure reason). + +# ERRORS + +ENOTSUP +: Completion counters are not supported on this device. + +# SEE ALSO + +**ibv_create_comp_cntr**(3), **ibv_qp_attach_comp_cntr**(3) + +# AUTHORS + +Michael Margolin diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c index bb5233b53..d91eb420a 100644 --- a/libibverbs/verbs.c +++ b/libibverbs/verbs.c @@ -792,6 +792,54 @@ LATEST_SYMVER_FUNC(ibv_ack_cq_events, 1_1, "IBVERBS_1.1", pthread_mutex_unlock(&cq->mutex); } +int _ibv_query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps, size_t caps_size) +{ + memset(caps, 0, caps_size); + return get_ops(context)->query_comp_cntr_caps(context, caps, caps_size); +} + +struct ibv_comp_cntr *ibv_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *cc_attr) +{ + return get_ops(context)->create_comp_cntr(context, cc_attr); +} + +int ibv_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr) +{ + return get_ops(comp_cntr->context)->destroy_comp_cntr(comp_cntr); +} + +int ibv_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value) +{ + return get_ops(comp_cntr->context)->set_comp_cntr(comp_cntr, value); +} + +int ibv_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value) +{ + return get_ops(comp_cntr->context)->set_err_comp_cntr(comp_cntr, value); +} + +int ibv_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount) +{ + return get_ops(comp_cntr->context)->inc_comp_cntr(comp_cntr, amount); +} + +int ibv_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount) +{ + return get_ops(comp_cntr->context)->inc_err_comp_cntr(comp_cntr, amount); +} + +int ibv_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value) +{ + return get_ops(comp_cntr->context)->read_comp_cntr(comp_cntr, value); +} + +int ibv_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value) +{ + return get_ops(comp_cntr->context)->read_err_comp_cntr(comp_cntr, value); +} + LATEST_SYMVER_FUNC(ibv_create_srq, 1_1, "IBVERBS_1.1", struct ibv_srq *, struct ibv_pd *pd, @@ -919,6 +967,12 @@ LATEST_SYMVER_FUNC(ibv_modify_qp, 1_1, "IBVERBS_1.1", return 0; } +int ibv_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr) +{ + return get_ops(qp->context)->qp_attach_comp_cntr(qp, comp_cntr, attr); +} + LATEST_SYMVER_FUNC(ibv_destroy_qp, 1_1, "IBVERBS_1.1", int, struct ibv_qp *qp) diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index 9ed55d423..cad28afdc 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -2117,6 +2117,42 @@ struct ibv_cq_init_attr_ex { struct ibv_pd *parent_domain; }; +enum ibv_qp_attach_comp_cntr_op { + IBV_QP_ATTACH_COMP_CNTR_OP_SEND = 1 << 0, + IBV_QP_ATTACH_COMP_CNTR_OP_RECV = 1 << 1, + IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ = 1 << 2, + IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ = 1 << 3, + IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE = 1 << 4, + IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE = 1 << 5, +}; + +struct ibv_comp_cntr_caps { + uint64_t max_value; + uint32_t max_counters; + uint32_t supported_qp_attach_ops; /* Bitmask of ibv_qp_attach_comp_cntr_op */ +}; + +struct ibv_comp_cntr { + struct ibv_context *context; + uint32_t handle; +}; + +enum ibv_comp_cntr_type { + IBV_COMP_CNTR_TYPE_WRS, + IBV_COMP_CNTR_TYPE_BYTES, +}; + +struct ibv_comp_cntr_init_attr { + uint32_t comp_mask; /* Compatibility mask */ + enum ibv_comp_cntr_type type; + uint32_t flags; +}; + +struct ibv_qp_attach_comp_cntr_attr { + uint32_t comp_mask; /* Compatibility mask */ + uint32_t op_mask; /* Bitmask of ibv_qp_attach_comp_cntr_op */ +}; + enum ibv_parent_domain_init_attr_mask { IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS = 1 << 0, IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT = 1 << 1, @@ -3027,6 +3063,77 @@ static inline int ibv_modify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *at return vctx->modify_cq(cq, attr); } + +int _ibv_query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps, size_t caps_size); + +/** + * ibv_query_comp_cntr_caps - Query completion counter capabilities + * @context: Device context. + * @caps: Output capabilities struct. + */ +static inline int ibv_query_comp_cntr_caps(struct ibv_context *context, + struct ibv_comp_cntr_caps *caps) +{ + return _ibv_query_comp_cntr_caps(context, caps, sizeof(*caps)); +} + +/** + * ibv_create_comp_cntr - Create a completion counter + * @context: Device context to create the counter on. + * @cc_attr: Attributes for the completion counter. + */ +struct ibv_comp_cntr *ibv_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *cc_attr); + +/** + * ibv_destroy_comp_cntr - Destroy a completion counter + * @comp_cntr: The completion counter to destroy. + */ +int ibv_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr); + +/** + * ibv_set_comp_cntr - Set the completion count value + * @comp_cntr: The completion counter to update. + * @value: The value to set. + */ +int ibv_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value); + +/** + * ibv_set_err_comp_cntr - Set the error count value + * @comp_cntr: The completion counter to update. + * @value: The value to set. + */ +int ibv_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value); + +/** + * ibv_inc_comp_cntr - Increment the completion count + * @comp_cntr: The completion counter to increment. + * @amount: The amount to increment by. + */ +int ibv_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount); + +/** + * ibv_inc_err_comp_cntr - Increment the error count + * @comp_cntr: The completion counter to increment. + * @amount: The amount to increment by. + */ +int ibv_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount); + +/** + * ibv_read_comp_cntr - Read the completion count value + * @comp_cntr: The completion counter to read. + * @value: Output pointer to store the current completion count. + */ +int ibv_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value); + +/** + * ibv_read_err_comp_cntr - Read the error count value + * @comp_cntr: The completion counter to read. + * @value: Output pointer to store the current error count. + */ +int ibv_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value); + /** * ibv_create_srq - Creates a SRQ associated with the specified protection * domain. @@ -3346,6 +3453,15 @@ ibv_modify_qp_rate_limit(struct ibv_qp *qp, return vctx->modify_qp_rate_limit(qp, attr); } +/** + * ibv_qp_attach_comp_cntr - Attach a completion counter to a QP + * @qp: The queue pair to attach the counter to. + * @comp_cntr: The completion counter to attach. + * @attr: Attach attributes. + */ +int ibv_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr); + /** * ibv_query_qp_data_in_order - Checks whether the data is guaranteed to be * written in-order. diff --git a/providers/efa/CMakeLists.txt b/providers/efa/CMakeLists.txt index 1f3c32c41..4054109c2 100644 --- a/providers/efa/CMakeLists.txt +++ b/providers/efa/CMakeLists.txt @@ -3,7 +3,7 @@ if (ENABLE_LTTNG AND LTTNGUST_FOUND) endif() rdma_shared_provider(efa libefa.map - 1 1.6.${PACKAGE_VERSION} + 1 1.7.${PACKAGE_VERSION} ${TRACE_FILE} efa.c verbs.c diff --git a/providers/efa/efa.c b/providers/efa/efa.c index 5e6d87371..7ef6ff3d2 100644 --- a/providers/efa/efa.c +++ b/providers/efa/efa.c @@ -31,30 +31,40 @@ static const struct verbs_context_ops efa_ctx_ops = { .alloc_pd = efa_alloc_pd, .alloc_parent_domain = efa_alloc_parent_domain, .alloc_td = efa_alloc_td, + .cq_event = efa_cq_event, .create_ah = efa_create_ah, + .create_comp_cntr = efa_create_comp_cntr, .create_cq = efa_create_cq, .create_cq_ex = efa_create_cq_ex, .create_qp = efa_create_qp, .create_qp_ex = efa_create_qp_ex, - .cq_event = efa_cq_event, .dealloc_pd = efa_dealloc_pd, .dealloc_td = efa_dealloc_td, .dereg_mr = efa_dereg_mr, .destroy_ah = efa_destroy_ah, + .destroy_comp_cntr = efa_destroy_comp_cntr, .destroy_cq = efa_destroy_cq, .destroy_qp = efa_destroy_qp, + .inc_comp_cntr = efa_inc_comp_cntr, + .inc_err_comp_cntr = efa_inc_err_comp_cntr, .modify_qp = efa_modify_qp, .poll_cq = efa_poll_cq, .post_recv = efa_post_recv, .post_send = efa_post_send, + .qp_attach_comp_cntr = efa_qp_attach_comp_cntr, + .query_comp_cntr_caps = ibv_cmd_query_comp_cntr_caps, .query_device_ex = efa_query_device_ex, .query_port = efa_query_port, .query_port_speed = efa_query_port_speed, .query_qp = efa_query_qp, .query_qp_data_in_order = efa_query_qp_data_in_order, + .read_comp_cntr = efa_read_comp_cntr, + .read_err_comp_cntr = efa_read_err_comp_cntr, .reg_dmabuf_mr = efa_reg_dmabuf_mr, .reg_mr = efa_reg_mr, .req_notify_cq = efa_arm_cq, + .set_comp_cntr = efa_set_comp_cntr, + .set_err_comp_cntr = efa_set_err_comp_cntr, .free_context = efa_free_context, }; diff --git a/providers/efa/efa.h b/providers/efa/efa.h index 7a596c9be..20567cc40 100644 --- a/providers/efa/efa.h +++ b/providers/efa/efa.h @@ -105,6 +105,14 @@ struct efa_cq { struct efa_sub_cq sub_cq_arr[]; }; +struct efa_comp_cntr { + struct ibv_comp_cntr ibv_comp_cntr; + uint64_t comp_val __attribute__((aligned(8))); + uint64_t err_val __attribute__((aligned(8))); + uint64_t *comp_ptr; + uint64_t *err_ptr; +}; + struct efa_wq { uint64_t *wrid; /* wrid_idx_pool: Pool of free indexes in the wrid array, used to select the @@ -230,6 +238,11 @@ static inline struct efa_cq *efadv_cq_to_efa_cq(struct efadv_cq *efadv_cq) return container_of(efadv_cq, struct efa_cq, dv_cq); } +static inline struct efa_comp_cntr *to_efa_comp_cntr(struct ibv_comp_cntr *ibvcc) +{ + return container_of(ibvcc, struct efa_comp_cntr, ibv_comp_cntr); +} + static inline struct efa_qp *to_efa_qp(struct ibv_qp *ibvqp) { return container_of(ibvqp, struct efa_qp, verbs_qp.qp); diff --git a/providers/efa/efadv.h b/providers/efa/efadv.h index 874598877..d863de480 100644 --- a/providers/efa/efadv.h +++ b/providers/efa/efadv.h @@ -23,6 +23,7 @@ enum { EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE = 1 << 3, EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV = 1 << 4, EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 5, + EFADV_DEVICE_ATTR_CAPS_COMP_CNTR = 1 << 6, }; struct efadv_device_attr { @@ -190,6 +191,40 @@ static inline bool efadv_wc_is_unsolicited(struct efadv_cq *efadv_cq) return efadv_cq->wc_is_unsolicited(efadv_cq); } +enum { + EFADV_MEMORY_LOCATION_VA, + EFADV_MEMORY_LOCATION_DMABUF, +}; + +struct efadv_memory_location { + uint8_t *ptr; + struct { + uint64_t offset; + int32_t fd; + uint32_t reserved; + } dmabuf; + uint8_t type; + uint8_t reserved[7]; +}; + +enum { + EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM = 1 << 0, + EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM = 1 << 1, +}; + +struct efadv_comp_cntr_init_attr { + uint64_t comp_mask; + uint32_t flags; + uint32_t reserved; + struct efadv_memory_location comp_cntr_ext_mem; + struct efadv_memory_location err_cntr_ext_mem; +}; + +struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen); + enum { EFADV_MR_ATTR_VALIDITY_RECV_IC_ID = 1 << 0, EFADV_MR_ATTR_VALIDITY_RDMA_READ_IC_ID = 1 << 1, diff --git a/providers/efa/libefa.map b/providers/efa/libefa.map index 3f51e5ee9..dee028a7a 100644 --- a/providers/efa/libefa.map +++ b/providers/efa/libefa.map @@ -40,3 +40,8 @@ EFA_1.6 { global: efadv_qp_from_ibv_qp_ex; } EFA_1.5; + +EFA_1.7 { + global: + efadv_create_comp_cntr; +} EFA_1.6; diff --git a/providers/efa/man/efadv_create_comp_cntr.3.md b/providers/efa/man/efadv_create_comp_cntr.3.md new file mode 100644 index 000000000..92d92dcbc --- /dev/null +++ b/providers/efa/man/efadv_create_comp_cntr.3.md @@ -0,0 +1,118 @@ +--- +layout: page +title: EFADV_CREATE_COMP_CNTR +section: 3 +tagline: Verbs +date: 2026-04-27 +header: "EFA Direct Verbs Manual" +footer: efa +--- + +# NAME + +efadv_create_comp_cntr - Create EFA specific Completion Counter + +# SYNOPSIS + +```c +#include + +struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen); +``` + +# DESCRIPTION + +**efadv_create_comp_cntr()** creates a Completion Counter with EFA specific +properties, such as external memory for the counter values. + +The argument *attr* is an ibv_comp_cntr_init_attr struct, as defined in +. + +Compatibility is handled using the comp_mask and inlen fields. + +```c +enum { + EFADV_MEMORY_LOCATION_VA, + EFADV_MEMORY_LOCATION_DMABUF, +}; + +struct efadv_memory_location { + uint8_t *ptr; + struct { + uint64_t offset; + int32_t fd; + uint32_t reserved; + } dmabuf; + uint8_t type; + uint8_t reserved[7]; +}; + +struct efadv_comp_cntr_init_attr { + uint64_t comp_mask; + uint32_t flags; + uint32_t reserved; + struct efadv_memory_location comp_cntr_ext_mem; + struct efadv_memory_location err_cntr_ext_mem; +}; +``` + +*inlen* +: In: Size of struct efadv_comp_cntr_init_attr. + +*comp_mask* +: Compatibility mask. + +*flags* +: A bitwise OR of the various values described below. + + **EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM**: + Use application-provided memory for the completion count, as + described by *comp_cntr_ext_mem*. + + **EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM**: + Use application-provided memory for the error count, as + described by *err_cntr_ext_mem*. + +*comp_cntr_ext_mem* +: Memory location for the completion count when using external memory. + +*err_cntr_ext_mem* +: Memory location for the error count when using external memory. + +## efadv_memory_location + +The external memory is described by an **efadv_memory_location** structure +which supports two modes: + +*type* +: **EFADV_MEMORY_LOCATION_VA** for a virtual address, or + **EFADV_MEMORY_LOCATION_DMABUF** for a DMA-BUF reference. + +*ptr* +: Virtual address pointer. Required when type is + **EFADV_MEMORY_LOCATION_VA**. When type is + **EFADV_MEMORY_LOCATION_DMABUF**, may optionally be set to provide a + process-accessible mapping of the DMA-BUF memory. + +*dmabuf.fd* +: DMA-BUF file descriptor (used when type is + **EFADV_MEMORY_LOCATION_DMABUF**). + +*dmabuf.offset* +: Offset within the DMA-BUF. + +# RETURN VALUE + +efadv_create_comp_cntr() returns a pointer to the created ibv_comp_cntr, or +NULL if the request fails. + +# SEE ALSO + +**efadv**(7), **ibv_create_comp_cntr**(3), **ibv_qp_attach_comp_cntr**(3) + +# AUTHORS + +Michael Margolin diff --git a/providers/efa/man/efadv_query_device.3.md b/providers/efa/man/efadv_query_device.3.md index f46d362fc..cff7068cd 100644 --- a/providers/efa/man/efadv_query_device.3.md +++ b/providers/efa/man/efadv_query_device.3.md @@ -92,6 +92,9 @@ struct efadv_device_attr { Indicates that creating CQs with external memory buffers by passing dmabuf is supported. + EFADV_DEVICE_ATTR_CAPS_COMP_CNTR: + Completion counters are supported. + *max_rdma_size* : Maximum RDMA transfer size in bytes. diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c index 8327cb301..aa76976ae 100644 --- a/providers/efa/verbs.c +++ b/providers/efa/verbs.c @@ -198,6 +198,9 @@ int efadv_query_device(struct ibv_context *ibvctx, if (EFA_DEV_CAP(ctx, CQ_WITH_EXT_MEM)) attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF; + + if (EFA_DEV_CAP(ctx, COMP_CNTR)) + attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_COMP_CNTR; } if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) { @@ -1404,6 +1407,183 @@ int efa_destroy_cq(struct ibv_cq *ibvcq) return 0; } +static void efa_fill_buffer_desc_va(struct ib_uverbs_buffer_desc *desc, uint64_t addr, + uint64_t length) +{ + desc->type = IB_UVERBS_BUFFER_TYPE_VA; + desc->addr = addr; + desc->length = length; +} + +static void efa_fill_buffer_desc_dmabuf(struct ib_uverbs_buffer_desc *desc, int32_t fd, + uint64_t offset, uint64_t length) +{ + desc->type = IB_UVERBS_BUFFER_TYPE_DMABUF; + desc->fd = fd; + desc->addr = offset; + desc->length = length; +} + +static void efa_fill_buffer_desc_from_mem_loc(struct ib_uverbs_buffer_desc *desc, + struct efadv_memory_location *mem, uint64_t length) +{ + if (mem->type == EFADV_MEMORY_LOCATION_DMABUF) + efa_fill_buffer_desc_dmabuf(desc, mem->dmabuf.fd, mem->dmabuf.offset, length); + else + efa_fill_buffer_desc_va(desc, (uintptr_t)mem->ptr, length); +} + +static inline bool efa_comp_cntr_mem_type_supported(uint32_t mem_type) +{ + return mem_type == EFADV_MEMORY_LOCATION_VA || mem_type == EFADV_MEMORY_LOCATION_DMABUF; +} + +static struct ibv_comp_cntr *efa_create_comp_cntr_impl(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr) +{ + uint32_t supported_efa_flags = EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM | + EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM; + DECLARE_COMMAND_BUFFER_LINK(cmdb, UVERBS_OBJECT_COMP_CNTR, + UVERBS_METHOD_COMP_CNTR_CREATE, 2, NULL); + struct ib_uverbs_buffer_desc comp_desc = {}; + struct ib_uverbs_buffer_desc err_desc = {}; + struct efa_comp_cntr *cc; + int err; + + if (attr->comp_mask || attr->flags || + attr->type != IBV_COMP_CNTR_TYPE_WRS || + efa_attr->comp_mask || + !check_comp_mask(efa_attr->flags, supported_efa_flags) || + !efa_comp_cntr_mem_type_supported(efa_attr->comp_cntr_ext_mem.type) || + !efa_comp_cntr_mem_type_supported(efa_attr->err_cntr_ext_mem.type)) { + verbs_err(verbs_get_ctx(ibvctx), "Unsupported type or flag\n"); + errno = EOPNOTSUPP; + return NULL; + } + + cc = calloc(1, sizeof(*cc)); + if (!cc) { + errno = ENOMEM; + return NULL; + } + + if (efa_attr->flags & EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM) { + efa_fill_buffer_desc_from_mem_loc(&comp_desc, &efa_attr->comp_cntr_ext_mem, + sizeof(uint64_t)); + cc->comp_ptr = (uint64_t *)efa_attr->comp_cntr_ext_mem.ptr; + } else { + efa_fill_buffer_desc_va(&comp_desc, (uintptr_t)&cc->comp_val, sizeof(uint64_t)); + cc->comp_ptr = &cc->comp_val; + } + + fill_attr_in_ptr(cmdb, EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, &comp_desc); + + if (efa_attr->flags & EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM) { + efa_fill_buffer_desc_from_mem_loc(&err_desc, &efa_attr->err_cntr_ext_mem, + sizeof(uint64_t)); + cc->err_ptr = (uint64_t *)efa_attr->err_cntr_ext_mem.ptr; + } else { + efa_fill_buffer_desc_va(&err_desc, (uintptr_t)&cc->err_val, sizeof(uint64_t)); + cc->err_ptr = &cc->err_val; + } + + fill_attr_in_ptr(cmdb, EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, &err_desc); + + err = ibv_cmd_create_comp_cntr(ibvctx, &cc->ibv_comp_cntr, cmdb); + if (err) { + free(cc); + errno = err; + return NULL; + } + + return &cc->ibv_comp_cntr; +} + +struct ibv_comp_cntr *efa_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr) +{ + struct efadv_comp_cntr_init_attr efa_attr = {}; + + return efa_create_comp_cntr_impl(ibvctx, attr, &efa_attr); +} + +struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen) +{ + if (!is_efa_dev(ibvctx->device)) { + verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n"); + errno = EOPNOTSUPP; + return NULL; + } + + if (!vext_field_avail(struct efadv_comp_cntr_init_attr, err_cntr_ext_mem, inlen) || + (inlen > sizeof(*efa_attr) && !is_ext_cleared(efa_attr, inlen))) { + verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n"); + errno = EINVAL; + return NULL; + } + + return efa_create_comp_cntr_impl(ibvctx, attr, efa_attr); +} + +int efa_destroy_comp_cntr(struct ibv_comp_cntr *ibvcc) +{ + struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc); + int err; + + err = ibv_cmd_destroy_comp_cntr(ibvcc); + if (err) + return err; + + free(cc); + return 0; +} + +int efa_set_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value) +{ + return ibv_cmd_set_comp_cntr(ibvcc, value); +} + +int efa_set_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value) +{ + return ibv_cmd_set_err_comp_cntr(ibvcc, value); +} + +int efa_inc_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount) +{ + return ibv_cmd_inc_comp_cntr(ibvcc, amount); +} + +int efa_inc_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount) +{ + return ibv_cmd_inc_err_comp_cntr(ibvcc, amount); +} + +int efa_read_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value) +{ + struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc); + + if (!cc->comp_ptr) + return EOPNOTSUPP; + + *value = *cc->comp_ptr; + return 0; +} + +int efa_read_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value) +{ + struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc); + + if (!cc->err_ptr) + return EOPNOTSUPP; + + *value = *cc->err_ptr; + return 0; +} + static void efa_wq_terminate(struct efa_wq *wq, int pgsz) { void *db_aligned; @@ -2145,6 +2325,12 @@ int efa_modify_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, return 0; } +int efa_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr) +{ + return ibv_cmd_qp_attach_comp_cntr(qp, comp_cntr, attr); +} + int efa_query_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr) { diff --git a/providers/efa/verbs.h b/providers/efa/verbs.h index b38b4e43b..602a8d1fa 100644 --- a/providers/efa/verbs.h +++ b/providers/efa/verbs.h @@ -36,12 +36,24 @@ int efa_poll_cq(struct ibv_cq *ibvcq, int nwc, struct ibv_wc *wc); int efa_arm_cq(struct ibv_cq *ibvcq, int solicited_only); void efa_cq_event(struct ibv_cq *ibvcq); +struct ibv_comp_cntr *efa_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr); +int efa_destroy_comp_cntr(struct ibv_comp_cntr *ibvcc); +int efa_set_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value); +int efa_set_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value); +int efa_inc_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount); +int efa_inc_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount); +int efa_read_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value); +int efa_read_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value); + struct ibv_qp *efa_create_qp(struct ibv_pd *ibvpd, struct ibv_qp_init_attr *attr); struct ibv_qp *efa_create_qp_ex(struct ibv_context *ibvctx, struct ibv_qp_init_attr_ex *attr_ex); int efa_modify_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int ibv_qp_attr_mask); +int efa_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr); int efa_query_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr); int efa_query_qp_data_in_order(struct ibv_qp *ibvqp, enum ibv_wr_opcode op, diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 4beb11843..6399fea71 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -22,6 +22,7 @@ rdma_cython_module(pyverbs "" addr.pyx base.pyx cmid.pyx + comp_cntr.pyx cq.pyx device.pyx ${DMA_UTIL} diff --git a/pyverbs/comp_cntr.pxd b/pyverbs/comp_cntr.pxd new file mode 100644 index 000000000..86997867c --- /dev/null +++ b/pyverbs/comp_cntr.pxd @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright Amazon.com, Inc. or its affiliates. All rights reserved. + +#cython: language_level=3 + +from pyverbs.base cimport PyverbsObject, PyverbsCM +from pyverbs.device cimport Context +cimport pyverbs.libibverbs as v + +cdef class CompCntrInitAttr(PyverbsObject): + cdef v.ibv_comp_cntr_init_attr attr + +cdef class CompCntr(PyverbsCM): + cdef v.ibv_comp_cntr *comp_cntr + cdef Context ctx + cpdef close(self) diff --git a/pyverbs/comp_cntr.pyx b/pyverbs/comp_cntr.pyx new file mode 100644 index 000000000..12e812cb9 --- /dev/null +++ b/pyverbs/comp_cntr.pyx @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright Amazon.com, Inc. or its affiliates. All rights reserved. + +#cython: language_level=3 + +from pyverbs.base import PyverbsRDMAErrno, PyverbsRDMAError +from pyverbs.device cimport Context +cimport pyverbs.libibverbs as v + + +cdef class CompCntrInitAttr(PyverbsObject): + """Represents ibv_comp_cntr_init_attr struct.""" + def __init__(self, comp_mask=0, cntr_type=0, flags=0): + super().__init__() + self.attr.comp_mask = comp_mask + self.attr.type = cntr_type + self.attr.flags = flags + + @property + def comp_mask(self): + return self.attr.comp_mask + + @property + def cntr_type(self): + return self.attr.type + + @property + def flags(self): + return self.attr.flags + + +cdef class CompCntr(PyverbsCM): + """Completion Counter object for tracking aggregate completions.""" + def __init__(self, Context ctx not None, CompCntrInitAttr attr not None): + """Create a completion counter. + + :param ctx: Device context to create the counter on. + :param attr: Completion counter init attributes. + """ + super().__init__() + self.comp_cntr = v.ibv_create_comp_cntr(ctx.context, &attr.attr) + if self.comp_cntr == NULL: + raise PyverbsRDMAErrno('Failed to create comp_cntr') + self.ctx = ctx + ctx.add_ref(self) + + def __dealloc__(self): + self.close() + + cpdef close(self): + """Destroy the completion counter.""" + if self.comp_cntr != NULL: + rc = v.ibv_destroy_comp_cntr(self.comp_cntr) + if rc: + raise PyverbsRDMAError('Failed to destroy comp_cntr', rc) + self.comp_cntr = NULL + + def set(self, value): + """Set the completion count to the given value.""" + rc = v.ibv_set_comp_cntr(self.comp_cntr, value) + if rc: + raise PyverbsRDMAError('Failed to set comp_cntr', rc) + + def set_err(self, value): + """Set the error count to the given value.""" + rc = v.ibv_set_err_comp_cntr(self.comp_cntr, value) + if rc: + raise PyverbsRDMAError('Failed to set_err comp_cntr', rc) + + def inc(self, amount): + """Increment the completion count by amount.""" + rc = v.ibv_inc_comp_cntr(self.comp_cntr, amount) + if rc: + raise PyverbsRDMAError('Failed to inc comp_cntr', rc) + + def inc_err(self, amount): + """Increment the error count by amount.""" + rc = v.ibv_inc_err_comp_cntr(self.comp_cntr, amount) + if rc: + raise PyverbsRDMAError('Failed to inc_err comp_cntr', rc) + + def read(self): + """Read and return the current completion count.""" + cdef unsigned long value = 0 + rc = v.ibv_read_comp_cntr(self.comp_cntr, &value) + if rc: + raise PyverbsRDMAError('Failed to read comp_cntr', rc) + return value + + def read_err(self): + """Read and return the current error count.""" + cdef unsigned long value = 0 + rc = v.ibv_read_err_comp_cntr(self.comp_cntr, &value) + if rc: + raise PyverbsRDMAError('Failed to read_err comp_cntr', rc) + return value diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 26bbe9a8e..02446cea6 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -23,6 +23,7 @@ cdef class Context(PyverbsCM): cdef object pps cdef object sched_nodes cdef object sched_leafs + cdef object comp_cntrs cdef object dr_domains cdef object wqs cdef object rwq_ind_tbls @@ -63,6 +64,9 @@ cdef class TSOCaps(PyverbsObject): cdef class DeviceAttrEx(PyverbsObject): cdef v.ibv_device_attr_ex dev_attr +cdef class CompCntrCaps(PyverbsObject): + cdef v.ibv_comp_cntr_caps caps + cdef class AllocDmAttr(PyverbsObject): cdef v.ibv_alloc_dm_attr alloc_dm_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 25466f1e9..ea1f71d32 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -9,6 +9,7 @@ which returns a DeviceAttr object. import weakref from .pyverbs_error import PyverbsRDMAError, PyverbsError +from pyverbs.comp_cntr cimport CompCntr from pyverbs.cq cimport CQEX, CQ, CompChannel from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno @@ -119,6 +120,7 @@ cdef class Context(PyverbsCM): self.pps = weakref.WeakSet() self.sched_nodes = weakref.WeakSet() self.sched_leafs = weakref.WeakSet() + self.comp_cntrs = weakref.WeakSet() self.dr_domains = weakref.WeakSet() self.wqs = weakref.WeakSet() self.rwq_ind_tbls = weakref.WeakSet() @@ -185,7 +187,8 @@ cdef class Context(PyverbsCM): self.crypto_logins, self.rwq_ind_tbls, self.wqs, self.ccs, self.cqs, self.dms, self.pds, self.xrcds, self.vars, self.sched_leafs, self.sched_nodes, - self.dr_domains, self.event_channels, self.dmahs]) + self.dr_domains, self.event_channels, self.dmahs, + self.comp_cntrs]) rc = v.ibv_close_device(self.context) if rc != 0: raise PyverbsRDMAErrno(f'Failed to close device {self.name}') @@ -228,6 +231,17 @@ cdef class Context(PyverbsCM): format(name=self.name), rc) return dev_attr_ex + def query_comp_cntr_caps(self): + """ + Query completion counter capabilities. + :return: CompCntrCaps object + """ + caps = CompCntrCaps() + rc = v.ibv_query_comp_cntr_caps(self.context, &caps.caps) + if rc != 0: + raise PyverbsRDMAError('Failed to query comp_cntr caps', rc) + return caps + def query_pkey(self, unsigned int port_num, int index): cdef uint16_t pkey rc = v.ibv_query_pkey(self.context, port_num, index, &pkey) @@ -366,6 +380,8 @@ cdef class Context(PyverbsCM): self.rwq_ind_tbls.add(obj) elif isinstance(obj, DMAHandle): self.dmahs.add(obj) + elif isinstance(obj, CompCntr): + self.comp_cntrs.add(obj) else: raise PyverbsError('Unrecognized object type') @@ -749,6 +765,21 @@ cdef class DeviceAttrEx(PyverbsObject): return self.dev_attr.phys_port_cnt_ex +cdef class CompCntrCaps(PyverbsObject): + """Completion counter capabilities.""" + @property + def max_counters(self): + return self.caps.max_counters + + @property + def max_value(self): + return self.caps.max_value + + @property + def supported_qp_attach_ops(self): + return self.caps.supported_qp_attach_ops + + cdef class AllocDmAttr(PyverbsObject): def __init__(self, length, log_align_req = 0, comp_mask = 0): """ diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index a8e942aba..82bfa869a 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -247,6 +247,22 @@ cdef extern from 'infiniband/verbs.h': cdef struct ibv_poll_cq_attr: unsigned int comp_mask + cdef struct ibv_comp_cntr_caps: + unsigned long max_value + unsigned int max_counters + unsigned int supported_qp_attach_ops + + cdef struct ibv_comp_cntr: + ibv_context *context + unsigned int handle + unsigned long comp_count_max_value + unsigned long err_count_max_value + + cdef struct ibv_comp_cntr_init_attr: + unsigned int comp_mask + unsigned int type + unsigned int flags + cdef struct ibv_wc_tm_info: unsigned long tag unsigned int priv @@ -512,6 +528,10 @@ cdef extern from 'infiniband/verbs.h': ibv_qp_type qp_type; unsigned int events_completed; + cdef struct ibv_qp_attach_comp_cntr_attr: + unsigned int comp_mask + unsigned int op_mask + cdef struct ibv_parent_domain_init_attr: ibv_pd *pd; uint32_t comp_mask; @@ -756,6 +776,17 @@ cdef extern from 'infiniband/verbs.h': unsigned int ibv_wc_read_flow_tag(ibv_cq_ex *cq) void ibv_wc_read_tm_info(ibv_cq_ex *cq, ibv_wc_tm_info *tm_info) unsigned long ibv_wc_read_completion_wallclock_ns(ibv_cq_ex *cq) + int ibv_query_comp_cntr_caps(ibv_context *context, + ibv_comp_cntr_caps *caps) + ibv_comp_cntr *ibv_create_comp_cntr(ibv_context *context, + ibv_comp_cntr_init_attr *attr) + int ibv_destroy_comp_cntr(ibv_comp_cntr *comp_cntr) + int ibv_set_comp_cntr(ibv_comp_cntr *comp_cntr, unsigned long value) + int ibv_set_err_comp_cntr(ibv_comp_cntr *comp_cntr, unsigned long value) + int ibv_inc_comp_cntr(ibv_comp_cntr *comp_cntr, unsigned long amount) + int ibv_inc_err_comp_cntr(ibv_comp_cntr *comp_cntr, unsigned long amount) + int ibv_read_comp_cntr(ibv_comp_cntr *comp_cntr, unsigned long *value) + int ibv_read_err_comp_cntr(ibv_comp_cntr *comp_cntr, unsigned long *value) ibv_ah *ibv_create_ah(ibv_pd *pd, ibv_ah_attr *attr) int ibv_init_ah_from_wc(ibv_context *context, uint8_t port_num, ibv_wc *wc, ibv_grh *grh, ibv_ah_attr *ah_attr) @@ -770,6 +801,8 @@ cdef extern from 'infiniband/verbs.h': int ibv_query_qp(ibv_qp *qp, ibv_qp_attr *attr, int attr_mask, ibv_qp_init_attr *init_attr) int ibv_destroy_qp(ibv_qp *qp) + int ibv_qp_attach_comp_cntr(ibv_qp *qp, ibv_comp_cntr *comp_cntr, + ibv_qp_attach_comp_cntr_attr *attr) int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr) int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr) int ibv_bind_mw(ibv_qp *qp, ibv_mw *mw, ibv_mw_bind *mw_bind) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index 9219a367d..c37f9fec3 100644 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -524,6 +524,14 @@ cdef extern from '': IBV_REG_MR_MASK_DMAH IBV_REG_MR_MASK_BUF + cpdef enum ibv_qp_attach_comp_cntr_op: + IBV_QP_ATTACH_COMP_CNTR_OP_SEND + IBV_QP_ATTACH_COMP_CNTR_OP_RECV + IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ + IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ + IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE + IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE + cdef extern from "": cdef unsigned long long IBV_ADVISE_MR_ADVICE_PREFETCH diff --git a/pyverbs/providers/efa/efa_enums.pxd b/pyverbs/providers/efa/efa_enums.pxd index 179f30cf5..725e486e1 100644 --- a/pyverbs/providers/efa/efa_enums.pxd +++ b/pyverbs/providers/efa/efa_enums.pxd @@ -11,6 +11,8 @@ cdef extern from 'infiniband/efadv.h': EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV + EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF + EFADV_DEVICE_ATTR_CAPS_COMP_CNTR cpdef enum: EFADV_QP_DRIVER_TYPE_SRD @@ -29,6 +31,14 @@ cdef extern from 'infiniband/efadv.h': EFADV_WC_EX_WITH_SGID EFADV_WC_EX_WITH_IS_UNSOLICITED + cpdef enum: + EFADV_MEMORY_LOCATION_VA + EFADV_MEMORY_LOCATION_DMABUF + + cpdef enum: + EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM + EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM + cpdef enum: EFADV_MR_ATTR_VALIDITY_RECV_IC_ID EFADV_MR_ATTR_VALIDITY_RDMA_READ_IC_ID diff --git a/pyverbs/providers/efa/efadv.pyx b/pyverbs/providers/efa/efadv.pyx index ab56f386a..791a3cb1a 100644 --- a/pyverbs/providers/efa/efadv.pyx +++ b/pyverbs/providers/efa/efadv.pyx @@ -6,12 +6,17 @@ cimport pyverbs.providers.efa.libefa as dv from pyverbs.addr cimport GID from pyverbs.base import PyverbsRDMAErrno, PyverbsRDMAError +from pyverbs.base cimport PyverbsCM +from pyverbs.comp_cntr cimport CompCntrInitAttr, CompCntr from pyverbs.cq cimport CQEX, CqInitAttrEx +from pyverbs.device cimport Context from pyverbs.libibverbs_enums import ibv_qp_attr_mask cimport pyverbs.libibverbs as v +from pyverbs.mr cimport MR from pyverbs.pd cimport PD from pyverbs.qp cimport QP, QPEx, QPInitAttr, QPInitAttrEx -from pyverbs.mr cimport MR +from libc.string cimport memset +from libc.stdint cimport uintptr_t, uint8_t def dev_cap_to_str(flags): @@ -21,6 +26,8 @@ def dev_cap_to_str(flags): dve.EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID: 'CQ entries with source GID', dve.EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE: 'RDMA Write', dve.EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV: 'Unsolicited RDMA Write receive', + dve.EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF: 'CQ with external memory', + dve.EFADV_DEVICE_ATTR_CAPS_COMP_CNTR: 'Completion Counters', } return bitmask_to_str(flags, l) @@ -408,3 +415,56 @@ cdef class EfaDVRQDepthAttr(PyverbsObject): @max_recv_sge.setter def max_recv_sge(self, val): self.rq_depth_attr.max_recv_sge = val + + +cdef class EfaCompCntrInitAttr(PyverbsObject): + """Represents efadv_comp_cntr_init_attr struct.""" + cdef dv.efadv_comp_cntr_init_attr attr + + def __init__(self, comp_ext_mem_ptr=None, err_ext_mem_ptr=None): + super().__init__() + memset(&self.attr, 0, sizeof(self.attr)) + if comp_ext_mem_ptr is not None: + self.attr.flags |= dve.EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM + self.attr.comp_cntr_ext_mem.type = dve.EFADV_MEMORY_LOCATION_VA + self.attr.comp_cntr_ext_mem.ptr = comp_ext_mem_ptr + if err_ext_mem_ptr is not None: + self.attr.flags |= dve.EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM + self.attr.err_cntr_ext_mem.type = dve.EFADV_MEMORY_LOCATION_VA + self.attr.err_cntr_ext_mem.ptr = err_ext_mem_ptr + + def set_comp_ext_mem_va(self, uintptr_t ptr): + """Set completion counter external memory VA.""" + self.attr.flags |= dve.EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM + self.attr.comp_cntr_ext_mem.type = dve.EFADV_MEMORY_LOCATION_VA + self.attr.comp_cntr_ext_mem.ptr = ptr + + def set_err_ext_mem_va(self, uintptr_t ptr): + """Set error counter external memory VA.""" + self.attr.flags |= dve.EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM + self.attr.err_cntr_ext_mem.type = dve.EFADV_MEMORY_LOCATION_VA + self.attr.err_cntr_ext_mem.ptr = ptr + + +cdef class EfaCompCntr(CompCntr): + """EFA-specific Completion Counter with external memory support.""" + def __init__(self, Context ctx not None, CompCntrInitAttr attr not None, + EfaCompCntrInitAttr efa_attr=None): + """Create an EFA completion counter. + + :param ctx: Device context. + :param attr: Completion counter init attributes. + :param efa_attr: EFA-specific init attributes (optional). + """ + PyverbsCM.__init__(self) + if efa_attr is None: + efa_attr = EfaCompCntrInitAttr() + self.comp_cntr = dv.efadv_create_comp_cntr( + ctx.context, &attr.attr, &efa_attr.attr, sizeof(efa_attr.attr)) + if self.comp_cntr == NULL: + raise PyverbsRDMAErrno('Failed to create EFA comp_cntr') + self.ctx = ctx + ctx.add_ref(self) + + def __dealloc__(self): + self.close() diff --git a/pyverbs/providers/efa/libefa.pxd b/pyverbs/providers/efa/libefa.pxd index 5cd4d1b06..878cc5b1a 100644 --- a/pyverbs/providers/efa/libefa.pxd +++ b/pyverbs/providers/efa/libefa.pxd @@ -86,3 +86,20 @@ cdef extern from 'infiniband/efadv.h': uint32_t inlen) int efadv_get_max_rq_depth(v.ibv_context *ibvctx, efadv_rq_depth_attr *attr, uint32_t inlen); + + cdef struct efadv_memory_location: + uint8_t *ptr + uint8_t type + uint8_t reserved[7] + + cdef struct efadv_comp_cntr_init_attr: + uint64_t comp_mask + uint32_t flags + uint32_t reserved + efadv_memory_location comp_cntr_ext_mem + efadv_memory_location err_cntr_ext_mem + + v.ibv_comp_cntr *efadv_create_comp_cntr(v.ibv_context *ibvctx, + v.ibv_comp_cntr_init_attr *attr, + efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen) diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd index 25afd325f..050d59df9 100644 --- a/pyverbs/qp.pxd +++ b/pyverbs/qp.pxd @@ -30,6 +30,9 @@ cdef class QPAttr(PyverbsObject): cdef class QPRateLimitAttr(PyverbsObject): cdef v.ibv_qp_rate_limit_attr attr +cdef class QPAttachCompCntrAttr(PyverbsObject): + cdef v.ibv_qp_attach_comp_cntr_attr attr + cdef class QP(PyverbsCM): cdef v.ibv_qp *qp cdef int type diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index b4f84033f..a14fd93eb 100644 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -21,6 +21,7 @@ from pyverbs.addr cimport GlobalRoute from pyverbs.device cimport Context from cpython.ref cimport PyObject from pyverbs.cq cimport CQ, CQEX +from pyverbs.comp_cntr cimport CompCntr cimport pyverbs.libibverbs as v from pyverbs.xrcd cimport XRCD from pyverbs.srq cimport SRQ @@ -978,6 +979,17 @@ cdef class QPRateLimitAttr(PyverbsObject): print_format.format('Comp mask', self.attr.comp_mask) +cdef class QPAttachCompCntrAttr(PyverbsObject): + """Attributes for attaching a completion counter to a QP.""" + def __init__(self, op_mask=0): + super().__init__() + self.attr.op_mask = op_mask + + @property + def op_mask(self): + return self.attr.op_mask + + cdef class QP(PyverbsCM): def __init__(self, object creator not None, object init_attr not None, QPAttr qp_attr=None): @@ -1235,6 +1247,18 @@ cdef class QP(PyverbsCM): if rc != 0: raise PyverbsRDMAError('Failed to modify QP rate limit', rc) + def attach_comp_cntr(self, CompCntr comp_cntr not None, + QPAttachCompCntrAttr attr not None): + """ + Attach a completion counter to this QP. + :param comp_cntr: The completion counter to attach + :param attr: Attach attributes including op_mask + """ + rc = v.ibv_qp_attach_comp_cntr(self.qp, comp_cntr.comp_cntr, + &attr.attr) + if rc != 0: + raise PyverbsRDMAError('Failed to attach comp cntr to QP', rc) + def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None): """ Post a receive WR on the QP.