diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 3697696ab..067c83d22 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols @@ -209,8 +209,10 @@ libhns.so.1 ibverbs-providers #MINVER# libmana.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev MANA_1.0@MANA_1.0 41 + MANA_1.1@MANA_1.1 65 manadv_init_obj@MANA_1.0 41 manadv_set_context_attr@MANA_1.0 41 + manadv_alloc_pd@MANA_1.1 65 libionic.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev IONIC_1.0@IONIC_1.0 59 diff --git a/kernel-headers/rdma/mana-abi.h b/kernel-headers/rdma/mana-abi.h index a75bf32b8..108795c43 100644 --- a/kernel-headers/rdma/mana-abi.h +++ b/kernel-headers/rdma/mana-abi.h @@ -57,6 +57,17 @@ struct mana_ib_create_rc_qp_resp { __u32 queue_id[4]; }; +struct mana_ib_create_uc_qp { + __aligned_u64 queue_buf[3]; + __u32 queue_size[3]; + __u32 comp_mask; +}; + +struct mana_ib_create_uc_qp_resp { + __u32 queue_id[3]; + __u32 reserved; +}; + struct mana_ib_create_wq { __aligned_u64 wq_buf_addr; __u32 wq_buf_size; @@ -87,4 +98,26 @@ struct mana_ib_create_qp_rss_resp { struct rss_resp_entry entries[64]; }; +enum mana_ib_ucontext_support { + MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0, +}; + +struct mana_ib_alloc_ucontext_resp { + __aligned_u64 comp_mask; +}; + +enum mana_ib_create_pd_flags { + MANA_IB_PD_SHORT_PDN = 1 << 0, +}; + +struct mana_ib_alloc_pd { + __u32 comp_mask; + __u32 reserved; +}; + +struct mana_ib_alloc_pd_resp { + __u32 pdn; + __u32 reserved; +}; + #endif diff --git a/kernel-headers/rdma/rdma_netlink.h b/kernel-headers/rdma/rdma_netlink.h index aac9782dd..3af946ecb 100644 --- a/kernel-headers/rdma/rdma_netlink.h +++ b/kernel-headers/rdma/rdma_netlink.h @@ -604,6 +604,11 @@ enum rdma_nldev_attr { RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES, /* u32 */ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY, /* u64 */ + /* + * Resource summary entry maximum value. + */ + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, /* u64 */ + /* * Always the end */ diff --git a/providers/mana/CMakeLists.txt b/providers/mana/CMakeLists.txt index 05011be02..3ae2b02ac 100644 --- a/providers/mana/CMakeLists.txt +++ b/providers/mana/CMakeLists.txt @@ -1,5 +1,5 @@ rdma_shared_provider(mana libmana.map - 1 1.0.${PACKAGE_VERSION} + 1 1.1.${PACKAGE_VERSION} mana.c manadv.c qp.c diff --git a/providers/mana/libmana.map b/providers/mana/libmana.map index ab66295d6..24af016fc 100644 --- a/providers/mana/libmana.map +++ b/providers/mana/libmana.map @@ -6,3 +6,8 @@ MANA_1.0 { manadv_init_obj; local: *; }; + +MANA_1.1 { + global: + manadv_alloc_pd; +} MANA_1.0; diff --git a/providers/mana/man/CMakeLists.txt b/providers/mana/man/CMakeLists.txt index 24f185969..3458443d7 100644 --- a/providers/mana/man/CMakeLists.txt +++ b/providers/mana/man/CMakeLists.txt @@ -2,4 +2,5 @@ rdma_man_pages( manadv.7.md manadv_init_obj.3.md manadv_set_context_attr.3.md + manadv_alloc_pd.3.md ) diff --git a/providers/mana/man/manadv_alloc_pd.3.md b/providers/mana/man/manadv_alloc_pd.3.md new file mode 100644 index 000000000..b121ea23c --- /dev/null +++ b/providers/mana/man/manadv_alloc_pd.3.md @@ -0,0 +1,36 @@ +--- +layout: page +title: manadv_alloc_pd +section: 3 +tagline: Verbs +--- + +# NAME +manadv_alloc_pd \- Create a MANA specific PD for the RDMA device context. + +# SYNOPSIS" +```c +#include + +struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags); +``` + +# DESCRIPTION +**manadv_alloc_pd()** allocates a PD for the RDMA device context with additional +creation flags. + +# ARGUMENTS +*context* +: RDMA device context to work on. + +*flags* +: A bitwise OR of the various values described below. + + MANADV_PD_FLAGS_SHORT_PDN: + allocates a PD with 16 bit PDN. + +# RETURN VALUE +returns a pointer to the allocated PD, or NULL if the request fails. + +# AUTHORS +Konstantin Taranov diff --git a/providers/mana/man/manadv_init_obj.3.md b/providers/mana/man/manadv_init_obj.3.md index 575ea34f7..a4dd7129b 100644 --- a/providers/mana/man/manadv_init_obj.3.md +++ b/providers/mana/man/manadv_init_obj.3.md @@ -48,6 +48,10 @@ struct manadv_rwq { void *db_page; }; +struct manadv_pd { + uint32_t pdn; +}; + struct manadv_obj { struct { struct ibv_qp *in; @@ -63,6 +67,11 @@ struct manadv_obj { struct ibv_wq *in; struct manadv_rwq *out; } rwq; + + struct { + struct ibv_pd *in; + struct manadv_pd *out; + } pd; }; ``` @@ -74,6 +83,7 @@ enum manadv_obj_type { MANADV_OBJ_QP = 1 << 0, MANADV_OBJ_CQ = 1 << 1, MANADV_OBJ_RWQ = 1 << 2, + MANADV_OBJ_PD = 1 << 3, }; ``` # RETURN VALUE diff --git a/providers/mana/mana.c b/providers/mana/mana.c index a59248b8f..87411d44c 100644 --- a/providers/mana/mana.c +++ b/providers/mana/mana.c @@ -21,9 +21,10 @@ #include "mana.h" DECLARE_DRV_CMD(mana_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty, - empty); + mana_ib_alloc_ucontext_resp); -DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, empty); +DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, mana_ib_alloc_pd, + mana_ib_alloc_pd_resp); static const struct verbs_match_ent hca_table[] = { VERBS_DRIVER_ID(RDMA_DRIVER_MANA), @@ -114,19 +115,34 @@ int mana_query_port(struct ibv_context *context, uint8_t port, return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); } -struct ibv_pd *mana_alloc_pd(struct ibv_context *context) +struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags) { - struct ibv_alloc_pd cmd; - struct mana_alloc_pd_resp resp; + struct mana_context *mctx = to_mctx(context); + struct mana_alloc_pd cmd = {}; + struct mana_ib_alloc_pd *cmd_drv = &cmd.drv_payload; + struct mana_alloc_pd_resp resp = {}; + size_t cmd_size = sizeof(cmd.ibv_cmd); /* v0 size */ struct mana_pd *pd; int ret; + if ((flags & MANADV_PD_FLAGS_SHORT_PDN) && + !(mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)) { + errno = EOPNOTSUPP; + return NULL; + } + pd = calloc(1, sizeof(*pd)); if (!pd) return NULL; - ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), - &resp.ibv_resp, sizeof(resp)); + if (mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT) + cmd_size = sizeof(cmd); /* v1 size */ + + if (flags & MANADV_PD_FLAGS_SHORT_PDN) + cmd_drv->comp_mask |= MANA_IB_PD_SHORT_PDN; + + ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd.ibv_cmd, cmd_size, + &resp.ibv_resp, sizeof(resp)); if (ret) { verbs_err(verbs_get_ctx(context), "Failed to allocate PD\n"); errno = ret; @@ -134,9 +150,16 @@ struct ibv_pd *mana_alloc_pd(struct ibv_context *context) return NULL; } + pd->pdn = resp.pdn; + return &pd->ibv_pd; } +static struct ibv_pd *mana_alloc_pd(struct ibv_context *context) +{ + return mana_alloc_pd_ex(context, 0); +} + struct ibv_pd * mana_alloc_parent_domain(struct ibv_context *context, struct ibv_parent_domain_init_attr *attr) @@ -452,7 +475,7 @@ static struct verbs_context *mana_alloc_context(struct ibv_device *ibdev, { int ret, i; struct mana_context *context; - struct mana_alloc_ucontext_resp resp; + struct mana_alloc_ucontext_resp resp = {}; struct ibv_get_context cmd; context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx, @@ -468,6 +491,8 @@ static struct verbs_context *mana_alloc_context(struct ibv_device *ibdev, goto free_ctx; } + context->comp_mask = resp.drv_payload.comp_mask; + verbs_set_ops(&context->ibv_ctx, &mana_ctx_ops); pthread_mutex_init(&context->qp_table_mutex, NULL); diff --git a/providers/mana/mana.h b/providers/mana/mana.h index 1f8a14718..e412e418b 100644 --- a/providers/mana/mana.h +++ b/providers/mana/mana.h @@ -89,6 +89,7 @@ struct mana_context { struct mana_table qp_rtable[MANA_QP_TABLE_SIZE]; struct mana_table qp_stable[MANA_QP_TABLE_SIZE]; pthread_mutex_t qp_table_mutex; + uint64_t comp_mask; struct manadv_ctx_allocators extern_alloc; void *db_page; @@ -196,6 +197,7 @@ struct mana_device { struct mana_pd { struct ibv_pd ibv_pd; + uint32_t pdn; struct mana_pd *mprotection_domain; }; @@ -216,7 +218,7 @@ int mana_query_device_ex(struct ibv_context *context, int mana_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr); -struct ibv_pd *mana_alloc_pd(struct ibv_context *context); +struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags); struct ibv_pd * mana_alloc_parent_domain(struct ibv_context *context, struct ibv_parent_domain_init_attr *attr); diff --git a/providers/mana/manadv.c b/providers/mana/manadv.c index 4b40d05bb..2392adf49 100644 --- a/providers/mana/manadv.c +++ b/providers/mana/manadv.c @@ -42,7 +42,7 @@ int manadv_set_context_attr(struct ibv_context *ibv_ctx, int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type) { - if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ)) + if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ | MANADV_OBJ_PD)) return EINVAL; if (obj_type & MANADV_OBJ_QP) { @@ -84,5 +84,17 @@ int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type) obj->rwq.out->db_page = ctx->db_page; } + if (obj_type & MANADV_OBJ_PD) { + struct ibv_pd *ibpd = obj->pd.in; + struct mana_pd *pd = container_of(ibpd, struct mana_pd, ibv_pd); + + obj->pd.out->pdn = pd->pdn; + } + return 0; } + +struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags) +{ + return mana_alloc_pd_ex(context, flags); +} diff --git a/providers/mana/manadv.h b/providers/mana/manadv.h index 27c8fe939..87f6e8f52 100644 --- a/providers/mana/manadv.h +++ b/providers/mana/manadv.h @@ -52,6 +52,10 @@ struct manadv_rwq { void *db_page; }; +struct manadv_pd { + uint32_t pdn; +}; + struct manadv_obj { struct { struct ibv_qp *in; @@ -67,16 +71,27 @@ struct manadv_obj { struct ibv_wq *in; struct manadv_rwq *out; } rwq; + struct { + struct ibv_pd *in; + struct manadv_pd *out; + } pd; }; enum manadv_obj_type { MANADV_OBJ_QP = 1 << 0, MANADV_OBJ_CQ = 1 << 1, MANADV_OBJ_RWQ = 1 << 2, + MANADV_OBJ_PD = 1 << 3, +}; + +enum { + MANADV_PD_FLAGS_SHORT_PDN = 1 << 0, }; int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type); +struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags); + #ifdef __cplusplus } #endif