providers/mana: expose PD number via manadv

debian/ibverbs-providers.symbols

@@ -209,8 +209,10 @@ libhns.so.1 ibverbs-providers #MINVER#
 libmana.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  MANA_1.0@MANA_1.0 41
+ MANA_1.1@MANA_1.1 65
  manadv_init_obj@MANA_1.0 41
  manadv_set_context_attr@MANA_1.0 41
+ manadv_alloc_pd@MANA_1.1 65
 libionic.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  IONIC_1.0@IONIC_1.0 59

kernel-headers/rdma/mana-abi.h

@@ -57,6 +57,17 @@ struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
 };
 
+struct mana_ib_create_uc_qp {
+	__aligned_u64 queue_buf[3];
+	__u32 queue_size[3];
+	__u32 comp_mask;
+};
+
+struct mana_ib_create_uc_qp_resp {
+	__u32 queue_id[3];
+	__u32 reserved;
+};
+
 struct mana_ib_create_wq {
 	__aligned_u64 wq_buf_addr;
 	__u32 wq_buf_size;
@@ -87,4 +98,26 @@ struct mana_ib_create_qp_rss_resp {
 	struct rss_resp_entry entries[64];
 };
 
+enum mana_ib_ucontext_support {
+	MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+};
+
+struct mana_ib_alloc_ucontext_resp {
+	__aligned_u64 comp_mask;
+};
+
+enum mana_ib_create_pd_flags {
+	MANA_IB_PD_SHORT_PDN = 1 << 0,
+};
+
+struct mana_ib_alloc_pd {
+	__u32 comp_mask;
+	__u32 reserved;
+};
+
+struct mana_ib_alloc_pd_resp {
+	__u32 pdn;
+	__u32 reserved;
+};
+
 #endif

kernel-headers/rdma/rdma_netlink.h

@@ -604,6 +604,11 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES,	/* u32 */
 	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY,	/* u64 */
 
+	/*
+	 * Resource summary entry maximum value.
+	 */
+	RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX,		/* u64 */
+
 	/*
 	 * Always the end
 	 */

providers/mana/CMakeLists.txt

@@ -1,5 +1,5 @@
 rdma_shared_provider(mana libmana.map
-  1 1.0.${PACKAGE_VERSION}
+  1 1.1.${PACKAGE_VERSION}
   mana.c
   manadv.c
   qp.c

providers/mana/libmana.map

@@ -6,3 +6,8 @@ MANA_1.0 {
 		manadv_init_obj;
 	local: *;
 };
+
+MANA_1.1 {
+	global:
+		manadv_alloc_pd;
+} MANA_1.0;

providers/mana/man/CMakeLists.txt

@@ -2,4 +2,5 @@ rdma_man_pages(
   manadv.7.md
   manadv_init_obj.3.md
   manadv_set_context_attr.3.md
+  manadv_alloc_pd.3.md
 )

providers/mana/man/manadv_alloc_pd.3.md

@@ -0,0 +1,36 @@
+---
+layout: page
+title: manadv_alloc_pd
+section: 3
+tagline: Verbs
+---
+
+# NAME
+manadv_alloc_pd \- Create a MANA specific PD for the RDMA device context.
+
+# SYNOPSIS"
+```c
+#include <infiniband/manadv.h>
+
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags);
+```
+
+# DESCRIPTION
+**manadv_alloc_pd()** allocates a PD for the RDMA device context with additional
+creation flags.
+
+# ARGUMENTS
+*context*
+:	RDMA device context to work on.
+
+*flags*
+:	A bitwise OR of the various values described below.
+
+	MANADV_PD_FLAGS_SHORT_PDN:
+		allocates a PD with 16 bit PDN.
+
+# RETURN VALUE
+returns a pointer to the allocated PD, or NULL if the request fails.
+
+# AUTHORS
+Konstantin Taranov <kotaranov@microsoft.com>

providers/mana/man/manadv_init_obj.3.md

@@ -48,6 +48,10 @@ struct manadv_rwq {
 	void		*db_page;
 };
 
+struct manadv_pd {
+	uint32_t	pdn;
+};
+
 struct manadv_obj {
 	struct {
 		struct ibv_qp		*in;
@@ -63,6 +67,11 @@ struct manadv_obj {
 		struct ibv_wq		*in;
 		struct manadv_rwq	*out;
 	} rwq;
+
+	struct {
+		struct ibv_pd		*in;
+		struct manadv_pd	*out;
+	} pd;
 };
 ```
 
@@ -74,6 +83,7 @@ enum manadv_obj_type {
 	MANADV_OBJ_QP   = 1 << 0,
 	MANADV_OBJ_CQ   = 1 << 1,
 	MANADV_OBJ_RWQ  = 1 << 2,
+	MANADV_OBJ_PD   = 1 << 3,
 };
 ```
 # RETURN VALUE

providers/mana/mana.c

@@ -21,9 +21,10 @@
 #include "mana.h"
 
 DECLARE_DRV_CMD(mana_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty,
-		empty);
+		mana_ib_alloc_ucontext_resp);
 
-DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, empty);
+DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, mana_ib_alloc_pd,
+		mana_ib_alloc_pd_resp);
 
 static const struct verbs_match_ent hca_table[] = {
 	VERBS_DRIVER_ID(RDMA_DRIVER_MANA),
@@ -114,29 +115,51 @@ int mana_query_port(struct ibv_context *context, uint8_t port,
 	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
 }
 
-struct ibv_pd *mana_alloc_pd(struct ibv_context *context)
+struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags)
 {
-	struct ibv_alloc_pd cmd;
-	struct mana_alloc_pd_resp resp;
+	struct mana_context *mctx = to_mctx(context);
+	struct mana_alloc_pd cmd = {};
+	struct mana_ib_alloc_pd *cmd_drv = &cmd.drv_payload;
+	struct mana_alloc_pd_resp resp = {};
+	size_t cmd_size = sizeof(cmd.ibv_cmd); /* v0 size */
 	struct mana_pd *pd;
 	int ret;
 
+	if ((flags & MANADV_PD_FLAGS_SHORT_PDN) &&
+	    !(mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)) {
+		errno = EOPNOTSUPP;
+		return NULL;
+	}
+
 	pd = calloc(1, sizeof(*pd));
 	if (!pd)
 		return NULL;
 
-	ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd),
-			       &resp.ibv_resp, sizeof(resp));
+	if (mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)
+		cmd_size = sizeof(cmd); /* v1 size */
+
+	if (flags & MANADV_PD_FLAGS_SHORT_PDN)
+		cmd_drv->comp_mask |= MANA_IB_PD_SHORT_PDN;
+
+	ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd.ibv_cmd, cmd_size,
+			      &resp.ibv_resp, sizeof(resp));
 	if (ret) {
 		verbs_err(verbs_get_ctx(context), "Failed to allocate PD\n");
 		errno = ret;
 		free(pd);
 		return NULL;
 	}
 
+	pd->pdn = resp.pdn;
+
 	return &pd->ibv_pd;
 }
 
+static struct ibv_pd *mana_alloc_pd(struct ibv_context *context)
+{
+	return mana_alloc_pd_ex(context, 0);
+}
+
 struct ibv_pd *
 mana_alloc_parent_domain(struct ibv_context *context,
 			 struct ibv_parent_domain_init_attr *attr)
@@ -452,7 +475,7 @@ static struct verbs_context *mana_alloc_context(struct ibv_device *ibdev,
 {
 	int ret, i;
 	struct mana_context *context;
-	struct mana_alloc_ucontext_resp resp;
+	struct mana_alloc_ucontext_resp resp = {};
 	struct ibv_get_context cmd;
 
 	context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
@@ -468,6 +491,8 @@ static struct verbs_context *mana_alloc_context(struct ibv_device *ibdev,
 		goto free_ctx;
 	}
 
+	context->comp_mask = resp.drv_payload.comp_mask;
+
 	verbs_set_ops(&context->ibv_ctx, &mana_ctx_ops);
 
 	pthread_mutex_init(&context->qp_table_mutex, NULL);

providers/mana/mana.h

@@ -89,6 +89,7 @@ struct mana_context {
 	struct mana_table qp_rtable[MANA_QP_TABLE_SIZE];
 	struct mana_table qp_stable[MANA_QP_TABLE_SIZE];
 	pthread_mutex_t qp_table_mutex;
+	uint64_t comp_mask;
 
 	struct manadv_ctx_allocators extern_alloc;
 	void *db_page;
@@ -196,6 +197,7 @@ struct mana_device {
 
 struct mana_pd {
 	struct ibv_pd ibv_pd;
+	uint32_t pdn;
 	struct mana_pd *mprotection_domain;
 };
 
@@ -216,7 +218,7 @@ int mana_query_device_ex(struct ibv_context *context,
 int mana_query_port(struct ibv_context *context, uint8_t port,
 		    struct ibv_port_attr *attr);
 
-struct ibv_pd *mana_alloc_pd(struct ibv_context *context);
+struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags);
 struct ibv_pd *
 mana_alloc_parent_domain(struct ibv_context *context,
 			 struct ibv_parent_domain_init_attr *attr);

providers/mana/manadv.c

@@ -42,7 +42,7 @@ int manadv_set_context_attr(struct ibv_context *ibv_ctx,
 
 int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type)
 {
-	if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ))
+	if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ | MANADV_OBJ_PD))
 		return EINVAL;
 
 	if (obj_type & MANADV_OBJ_QP) {
@@ -84,5 +84,17 @@ int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type)
 		obj->rwq.out->db_page = ctx->db_page;
 	}
 
+	if (obj_type & MANADV_OBJ_PD) {
+		struct ibv_pd *ibpd = obj->pd.in;
+		struct mana_pd *pd = container_of(ibpd, struct mana_pd, ibv_pd);
+
+		obj->pd.out->pdn = pd->pdn;
+	}
+
 	return 0;
 }
+
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags)
+{
+	return mana_alloc_pd_ex(context, flags);
+}

providers/mana/manadv.h

@@ -52,6 +52,10 @@ struct manadv_rwq {
 	void *db_page;
 };
 
+struct manadv_pd {
+	uint32_t pdn;
+};
+
 struct manadv_obj {
 	struct {
 		struct ibv_qp *in;
@@ -67,16 +71,27 @@ struct manadv_obj {
 		struct ibv_wq *in;
 		struct manadv_rwq *out;
 	} rwq;
+	struct {
+		struct ibv_pd *in;
+		struct manadv_pd *out;
+	} pd;
 };
 
 enum manadv_obj_type {
 	MANADV_OBJ_QP = 1 << 0,
 	MANADV_OBJ_CQ = 1 << 1,
 	MANADV_OBJ_RWQ = 1 << 2,
+	MANADV_OBJ_PD = 1 << 3,
+};
+
+enum {
+	MANADV_PD_FLAGS_SHORT_PDN = 1 << 0,
 };
 
 int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type);
 
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags);
+
 #ifdef __cplusplus
 }
 #endif