providers/mana: UC QP support

kernel-headers/rdma/mana-abi.h

@@ -57,6 +57,17 @@ struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
 };
 
+struct mana_ib_create_uc_qp {
+	__aligned_u64 queue_buf[3];
+	__u32 queue_size[3];
+	__u32 comp_mask;
+};
+
+struct mana_ib_create_uc_qp_resp {
+	__u32 queue_id[3];
+	__u32 reserved;
+};
+
 struct mana_ib_create_wq {
 	__aligned_u64 wq_buf_addr;
 	__u32 wq_buf_size;
@@ -87,4 +98,26 @@ struct mana_ib_create_qp_rss_resp {
 	struct rss_resp_entry entries[64];
 };
 
+enum mana_ib_ucontext_support {
+	MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+};
+
+struct mana_ib_alloc_ucontext_resp {
+	__aligned_u64 comp_mask;
+};
+
+enum mana_ib_create_pd_flags {
+	MANA_IB_PD_SHORT_PDN = 1 << 0,
+};
+
+struct mana_ib_alloc_pd {
+	__u32 comp_mask;
+	__u32 reserved;
+};
+
+struct mana_ib_alloc_pd_resp {
+	__u32 pdn;
+	__u32 reserved;
+};
+
 #endif

kernel-headers/rdma/rdma_netlink.h

@@ -604,6 +604,11 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES,	/* u32 */
 	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY,	/* u64 */
 
+	/*
+	 * Resource summary entry maximum value.
+	 */
+	RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX,		/* u64 */
+
 	/*
 	 * Always the end
 	 */

providers/mana/cq.c

@@ -284,6 +284,25 @@ static inline int advance_send_completions(struct mana_qp *qp, uint32_t psn,
 	return produced;
 }
 
+static inline int handle_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe, struct ibv_wc *wc)
+{
+	struct mana_gdma_queue *send_queue = mana_ib_get_sreq(qp);
+	struct shadow_wqe_header *wqe;
+	int produced = 0;
+
+	while (!produced && (wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) != NULL) {
+		send_queue->cons_idx += wqe->posted_wqe_size_in_bu;
+		send_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+		if (wqe->flags != MANA_NO_SIGNAL_WC) {
+			fill_verbs_from_shadow_wqe(qp, wc, wqe);
+			produced++;
+		}
+		shadow_queue_advance_consumer(&qp->shadow_sq);
+	}
+
+	return produced;
+}
+
 static inline int handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe,
 					  struct ibv_wc *wc, int nwc, bool *consumed)
 {
@@ -443,6 +462,8 @@ static inline int mana_handle_cqe(struct mana_context *ctx, struct gdma_cqe *cqe
 		return handle_error_cqe(qp, cqe, wc, nwc, consumed);
 	else if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL)
 		return handle_rc_requester_cqe(qp, cqe, wc, nwc, consumed);
+	else if (cqe->is_sq && cqe->rdma_cqe.cqe_type == CQE_TYPE_UD_SEND)
+		return handle_requester_cqe(qp, cqe, wc);
 	else
 		return handle_responder_cqe(qp, cqe, wc);
 }

providers/mana/mana.h

@@ -49,7 +49,8 @@ enum user_queue_types {
 	USER_RNIC_SEND_QUEUE_RESPONDER = 1,
 	USER_RNIC_RECV_QUEUE_REQUESTER = 2,
 	USER_RNIC_RECV_QUEUE_RESPONDER = 3,
-	USER_RNIC_QUEUE_TYPE_MAX = 4,
+	USER_RNIC_SEND_QUEUE_MM = 4,
+	USER_RNIC_QUEUE_TYPE_MAX = 5,
 };
 
 #define QUEUE_TYPE_MASK 0x3

providers/mana/qp.c

@@ -31,6 +31,9 @@ DECLARE_DRV_CMD(mana_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
 DECLARE_DRV_CMD(mana_create_rc_qp, IB_USER_VERBS_CMD_CREATE_QP,
 		mana_ib_create_rc_qp, mana_ib_create_rc_qp_resp);
 
+DECLARE_DRV_CMD(mana_create_uc_qp, IB_USER_VERBS_CMD_CREATE_QP,
+		mana_ib_create_uc_qp, mana_ib_create_uc_qp_resp);
+
 static struct ibv_qp *mana_create_qp_raw(struct ibv_pd *ibpd,
 					 struct ibv_qp_init_attr *attr)
 {
@@ -216,56 +219,149 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty
 	uint32_t size = 0;
 	uint32_t sges = 0;
 
-	if (attr->qp_type == IBV_QPT_RC) {
-		switch (type) {
-		case USER_RNIC_SEND_QUEUE_REQUESTER:
-			/* WQE must have at least one SGE */
-			/* For write with imm we need one extra SGE */
-			sges = max(1U, attr->cap.max_send_sge) + 1;
-			size = attr->cap.max_send_wr * get_large_wqe_size(sges);
-			break;
-		case USER_RNIC_SEND_QUEUE_RESPONDER:
-			size = MANA_PAGE_SIZE;
-			break;
-		case USER_RNIC_RECV_QUEUE_REQUESTER:
-			size = MANA_PAGE_SIZE;
-			break;
-		case USER_RNIC_RECV_QUEUE_RESPONDER:
-			/* WQE must have at least one SGE */
-			sges = max(1U, attr->cap.max_recv_sge);
-			size = attr->cap.max_recv_wr * get_wqe_size(sges);
-			break;
-		default:
-			return 0;
-		}
+	switch (type) {
+	case USER_RNIC_SEND_QUEUE_REQUESTER:
+		/* WQE must have at least one SGE */
+		/* For write with imm we need one extra SGE */
+		sges = max(1U, attr->cap.max_send_sge) + 1;
+		size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+		break;
+	case USER_RNIC_SEND_QUEUE_RESPONDER:
+		if (attr->qp_type == IBV_QPT_RC)
+			size = align_hw_size(MANA_PAGE_SIZE);
+		break;
+	case USER_RNIC_RECV_QUEUE_REQUESTER:
+		if (attr->qp_type == IBV_QPT_RC)
+			size = align_hw_size(MANA_PAGE_SIZE);
+		break;
+	case USER_RNIC_RECV_QUEUE_RESPONDER:
+		/* WQE must have at least one SGE */
+		sges = max(1U, attr->cap.max_recv_sge);
+		size = align_hw_size(attr->cap.max_recv_wr * get_wqe_size(sges));
+		break;
+	case USER_RNIC_SEND_QUEUE_MM:
+		sges = max(1U, attr->cap.max_send_sge) + 1;
+		size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+		break;
+	default:
+		return 0;
 	}
 
-	size = align_hw_size(size);
-
 	if (attr->qp_type == IBV_QPT_RC && type == USER_RNIC_SEND_QUEUE_REQUESTER)
 		size += sizeof(struct mana_ib_rollback_shared_mem);
 
 	return size;
 }
 
-static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
-					  struct ibv_qp_init_attr *attr)
+static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
+				 struct ibv_qp_init_attr *attr)
 {
-	struct mana_context *ctx = to_mctx(ibpd->context);
 	struct mana_ib_create_rc_qp_resp *qp_resp_drv;
 	struct mana_create_rc_qp_resp qp_resp = {};
 	struct mana_ib_create_rc_qp *qp_cmd_drv;
 	struct mana_create_rc_qp qp_cmd = {};
+	int ret, i;
+
+	qp_cmd_drv = &qp_cmd.drv_payload;
+	qp_resp_drv = &qp_resp.drv_payload;
+
+	for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+		if (i == USER_RNIC_SEND_QUEUE_MM)
+			continue;
+		qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
+		qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
+	}
+
+	ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+				sizeof(qp_cmd), &qp_resp.ibv_resp,
+				sizeof(qp_resp));
+	if (ret) {
+		verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+		return ret;
+	}
+
+	for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+		if (i == USER_RNIC_SEND_QUEUE_MM)
+			continue;
+		qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
+	}
+
+	return 0;
+}
+
+enum {
+	MANA_UC_UDATA_SQR = 0,
+	MANA_UC_UDATA_RQR = 1,
+	MANA_UC_UDATA_SMQ = 2,
+};
+
+static int mana_create_cmd_qp_uc(struct mana_qp *qp, struct ibv_pd *ibpd,
+				 struct ibv_qp_init_attr *attr)
+{
+	struct mana_ib_create_uc_qp_resp *qp_resp_drv;
+	struct mana_create_uc_qp_resp qp_resp = {};
+	struct mana_ib_create_uc_qp *qp_cmd_drv;
+	struct mana_create_uc_qp qp_cmd = {};
+	int ret;
+
+	qp_cmd_drv = &qp_cmd.drv_payload;
+	qp_resp_drv = &qp_resp.drv_payload;
+
+	qp_cmd_drv->queue_buf[MANA_UC_UDATA_SQR] =
+		(uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].buffer;
+	qp_cmd_drv->queue_size[MANA_UC_UDATA_SQR] =
+		qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].size;
+
+	qp_cmd_drv->queue_buf[MANA_UC_UDATA_RQR] =
+		(uintptr_t)qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].buffer;
+	qp_cmd_drv->queue_size[MANA_UC_UDATA_RQR] =
+		qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].size;
+
+	qp_cmd_drv->queue_buf[MANA_UC_UDATA_SMQ] =
+		(uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].buffer;
+	qp_cmd_drv->queue_size[MANA_UC_UDATA_SMQ] =
+		qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].size;
+
+	ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+				sizeof(qp_cmd), &qp_resp.ibv_resp,
+				sizeof(qp_resp));
+	if (ret) {
+		verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+		return ret;
+	}
+
+	qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].id =
+		qp_resp_drv->queue_id[MANA_UC_UDATA_SQR];
+	qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id =
+		qp_resp_drv->queue_id[MANA_UC_UDATA_RQR];
+	qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].id =
+		qp_resp_drv->queue_id[MANA_UC_UDATA_SMQ];
+
+	return 0;
+}
+
+static int mana_create_cmd_qp(struct mana_qp *qp, struct ibv_pd *ibpd,
+			      struct ibv_qp_init_attr *attr)
+{
+	if (attr->qp_type == IBV_QPT_RC)
+		return mana_create_cmd_qp_rc(qp, ibpd, attr);
+	else if (attr->qp_type == IBV_QPT_UC)
+		return mana_create_cmd_qp_uc(qp, ibpd, attr);
+	else
+		return -EOPNOTSUPP;
+}
+
+static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
+					  struct ibv_qp_init_attr *attr)
+{
+	struct mana_context *ctx = to_mctx(ibpd->context);
 	struct mana_qp *qp;
 	int ret, i;
 
 	qp = calloc(1, sizeof(*qp));
 	if (!qp)
 		return NULL;
 
-	qp_cmd_drv = &qp_cmd.drv_payload;
-	qp_resp_drv = &qp_resp.drv_payload;
-
 	pthread_spin_init(&qp->sq_lock, PTHREAD_PROCESS_PRIVATE);
 	pthread_spin_init(&qp->rq_lock, PTHREAD_PROCESS_PRIVATE);
 	qp->sq_sig_all = attr->sq_sig_all;
@@ -291,29 +387,18 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
 
 		if (qp->rnic_qp.queues[i].size != 0 && !qp->rnic_qp.queues[i].buffer) {
 			verbs_err(verbs_get_ctx(ibpd->context),
-				  "Failed to allocate memory for RC queue %d\n", i);
+				  "Failed to allocate memory for queue %d\n", i);
 			errno = ENOMEM;
 			goto destroy_queues;
 		}
-
-		qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
-		qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
 	}
 
-	mana_ib_init_rb_shmem(qp);
-
-	ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
-				sizeof(qp_cmd), &qp_resp.ibv_resp,
-				sizeof(qp_resp));
+	ret = mana_create_cmd_qp(qp, ibpd, attr);
 	if (ret) {
-		verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
 		errno = ret;
-		goto free_rb;
+		goto destroy_queues;
 	}
 
-	for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i)
-		qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
-
 	qp->ibqp.qp.qp_num = qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id;
 
 	ret = mana_store_qp(ctx, qp);
@@ -322,12 +407,12 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
 		goto destroy_qp;
 	}
 
+	mana_ib_init_rb_shmem(qp);
+
 	return &qp->ibqp.qp;
 
 destroy_qp:
 	ibv_cmd_destroy_qp(&qp->ibqp.qp);
-free_rb:
-	mana_ib_deinit_rb_shmem(qp);
 destroy_queues:
 	while (i-- > 0)
 		mana_dealloc_mem(qp->rnic_qp.queues[i].buffer, qp->rnic_qp.queues[i].size);
@@ -346,6 +431,7 @@ struct ibv_qp *mana_create_qp(struct ibv_pd *ibpd,
 	case IBV_QPT_RAW_PACKET:
 		return mana_create_qp_raw(ibpd, attr);
 	case IBV_QPT_RC:
+	case IBV_QPT_UC:
 		return mana_create_qp_rnic(ibpd, attr);
 	default:
 		verbs_err(verbs_get_ctx(ibpd->context),
@@ -382,7 +468,8 @@ static void mana_ib_modify_rnic_qp(struct mana_qp *qp, struct ibv_qp_attr *attr,
 			if (attr_mask & IBV_QP_SQ_PSN) {
 				qp->sq_ssn = 1;
 				qp->sq_psn = attr->sq_psn;
-				gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
+				if (qp->ibqp.qp.qp_type == IBV_QPT_RC)
+					gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
 			}
 			break;
 		default:
@@ -397,7 +484,7 @@ int mana_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask)
 	struct ibv_modify_qp cmd = {};
 	int err;
 
-	if (ibqp->qp_type != IBV_QPT_RC)
+	if (ibqp->qp_type != IBV_QPT_RC && ibqp->qp_type != IBV_QPT_UC)
 		return EOPNOTSUPP;
 
 	pthread_spin_lock(&qp->sq_lock);
@@ -443,7 +530,7 @@ int mana_destroy_qp(struct ibv_qp *ibqp)
 	struct mana_context *ctx = to_mctx(ibqp->context);
 	int ret, i;
 
-	if (ibqp->qp_type == IBV_QPT_RC) {
+	if (ibqp->qp_type == IBV_QPT_RC || ibqp->qp_type == IBV_QPT_UC) {
 		mana_remove_qp(ctx, qp);
 		mana_drain_cqes(qp);
 	}
@@ -459,6 +546,7 @@ int mana_destroy_qp(struct ibv_qp *ibqp)
 		ctx->extern_alloc.free(qp->raw_qp.send_buf, ctx->extern_alloc.data);
 		break;
 	case IBV_QPT_RC:
+	case IBV_QPT_UC:
 		pthread_spin_destroy(&qp->sq_lock);
 		pthread_spin_destroy(&qp->rq_lock);
 		destroy_shadow_queue(&qp->shadow_sq);