bnxt_re/lib: Support direct verbs

kernel-headers/rdma/bnxt_re-abi.h

@@ -162,6 +162,10 @@ enum bnxt_re_objects {
 	BNXT_RE_OBJECT_ALLOC_PAGE = (1U << UVERBS_ID_NS_SHIFT),
 	BNXT_RE_OBJECT_NOTIFY_DRV,
 	BNXT_RE_OBJECT_GET_TOGGLE_MEM,
+	BNXT_RE_OBJECT_DBR,
+	BNXT_RE_OBJECT_UMEM,
+	BNXT_RE_OBJECT_DV_CQ,
+	BNXT_RE_OBJECT_DV_QP,
 };
 
 enum bnxt_re_alloc_page_type {
@@ -215,4 +219,142 @@ enum bnxt_re_toggle_mem_methods {
 	BNXT_RE_METHOD_GET_TOGGLE_MEM = (1U << UVERBS_ID_NS_SHIFT),
 	BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
 };
+
+struct bnxt_re_dv_db_region {
+	__u32 dbr_handle;
+	__u32 dpi;
+	__u64 umdbr;
+	void *dbr;
+	__aligned_u64 comp_mask;
+};
+
+enum bnxt_re_obj_dbr_alloc_attrs {
+	BNXT_RE_DV_ALLOC_DBR_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_DV_ALLOC_DBR_ATTR,
+	BNXT_RE_DV_ALLOC_DBR_OFFSET,
+};
+
+enum bnxt_re_obj_dbr_free_attrs {
+	BNXT_RE_DV_FREE_DBR_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum bnxt_re_obj_dbr_query_attrs {
+	BNXT_RE_DV_QUERY_DBR_ATTR = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum bnxt_re_obj_dpi_methods {
+	BNXT_RE_METHOD_DBR_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_METHOD_DBR_FREE,
+	BNXT_RE_METHOD_DBR_QUERY,
+};
+
+enum bnxt_re_dv_umem_reg_attrs {
+	BNXT_RE_UMEM_OBJ_REG_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_UMEM_OBJ_REG_ADDR,
+	BNXT_RE_UMEM_OBJ_REG_LEN,
+	BNXT_RE_UMEM_OBJ_REG_ACCESS,
+	BNXT_RE_UMEM_OBJ_REG_DMABUF_FD,
+	BNXT_RE_UMEM_OBJ_REG_PGSZ_BITMAP,
+};
+
+enum bnxt_re_dv_umem_dereg_attrs {
+	BNXT_RE_UMEM_OBJ_DEREG_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum bnxt_re_dv_umem_methods {
+	BNXT_RE_METHOD_UMEM_REG = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_METHOD_UMEM_DEREG,
+};
+
+struct bnxt_re_dv_cq_req {
+	__u32 ncqe;
+	__aligned_u64 va;
+	__aligned_u64 comp_mask;
+};
+
+struct bnxt_re_dv_cq_resp {
+	__u32 cqid;
+	__u32 tail;
+	__u32 phase;
+	__u32 rsvd;
+	__aligned_u64 comp_mask;
+};
+
+enum bnxt_re_dv_create_cq_attrs {
+	BNXT_RE_DV_CREATE_CQ_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_DV_CREATE_CQ_REQ,
+	BNXT_RE_DV_CREATE_CQ_UMEM_HANDLE,
+	BNXT_RE_DV_CREATE_CQ_UMEM_OFFSET,
+	BNXT_RE_DV_CREATE_CQ_RESP,
+};
+
+enum bnxt_re_dv_cq_methods {
+	BNXT_RE_METHOD_DV_CREATE_CQ = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_METHOD_DV_DESTROY_CQ,
+};
+
+enum bnxt_re_dv_destroy_cq_attrs {
+	BNXT_RE_DV_DESTROY_CQ_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+struct bnxt_re_dv_create_qp_req {
+	int qp_type;
+	__u32 max_send_wr;
+	__u32 max_recv_wr;
+	__u32 max_send_sge;
+	__u32 max_recv_sge;
+	__u32 max_inline_data;
+	__u32 pd_id;
+	__aligned_u64 qp_handle;
+	__aligned_u64 sq_va;
+	__u32 sq_umem_offset;
+	__u32 sq_len;   /* total len including MSN area */
+	__u32 sq_slots;
+	__u32 sq_wqe_sz;
+	__u32 sq_psn_sz;
+	__u32 sq_npsn;
+	__aligned_u64 rq_va;
+	__u32 rq_umem_offset;
+	__u32 rq_len;
+	__u32 rq_slots; /* == max_recv_wr */
+	__u32 rq_wqe_sz;
+};
+
+struct bnxt_re_dv_create_qp_resp {
+	__u32 qpid;
+};
+
+enum bnxt_re_dv_create_qp_attrs {
+	BNXT_RE_DV_CREATE_QP_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_DV_CREATE_QP_REQ,
+	BNXT_RE_DV_CREATE_QP_SEND_CQ_HANDLE,
+	BNXT_RE_DV_CREATE_QP_RECV_CQ_HANDLE,
+	BNXT_RE_DV_CREATE_QP_SQ_UMEM_HANDLE,
+	BNXT_RE_DV_CREATE_QP_RQ_UMEM_HANDLE,
+	BNXT_RE_DV_CREATE_QP_SRQ_HANDLE,
+	BNXT_RE_DV_CREATE_QP_DBR_HANDLE,
+	BNXT_RE_DV_CREATE_QP_RESP
+};
+
+enum bnxt_re_dv_qp_methods {
+	BNXT_RE_METHOD_DV_CREATE_QP = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_METHOD_DV_DESTROY_QP,
+	BNXT_RE_METHOD_DV_MODIFY_QP,
+	BNXT_RE_METHOD_DV_QUERY_QP,
+};
+
+enum bnxt_re_dv_destroy_qp_attrs {
+	BNXT_RE_DV_DESTROY_QP_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum bnxt_re_var_dv_modify_qp_attrs {
+	BNXT_RE_DV_MODIFY_QP_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_DV_MODIFY_QP_REQ,
+};
+
+enum bnxt_re_dv_query_qp_attrs {
+	BNXT_RE_DV_QUERY_QP_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_DV_QUERY_QP_ATTR,
+};
+
 #endif /* __BNXT_RE_UVERBS_ABI_H__*/

kernel-headers/rdma/irdma-abi.h

@@ -20,11 +20,14 @@ enum irdma_memreg_type {
 	IRDMA_MEMREG_TYPE_MEM  = 0,
 	IRDMA_MEMREG_TYPE_QP   = 1,
 	IRDMA_MEMREG_TYPE_CQ   = 2,
+	IRDMA_MEMREG_TYPE_SRQ  = 3,
 };
 
 enum {
 	IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
 	IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
+	IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA = 1 << 2,
+	IRDMA_SUPPORT_WQE_FORMAT_V2 = 1 << 3,
 };
 
 struct irdma_alloc_ucontext_req {
@@ -54,7 +57,8 @@ struct irdma_alloc_ucontext_resp {
 	__u8 rsvd2;
 	__aligned_u64 comp_mask;
 	__u16 min_hw_wq_size;
-	__u8 rsvd3[6];
+	__u32 max_hw_srq_quanta;
+	__u8 rsvd3[2];
 };
 
 struct irdma_alloc_pd_resp {
@@ -71,6 +75,16 @@ struct irdma_create_cq_req {
 	__aligned_u64 user_shadow_area;
 };
 
+struct irdma_create_srq_req {
+	__aligned_u64 user_srq_buf;
+	__aligned_u64 user_shadow_area;
+};
+
+struct irdma_create_srq_resp {
+	__u32 srq_id;
+	__u32 srq_size;
+};
+
 struct irdma_create_qp_req {
 	__aligned_u64 user_wqe_bufs;
 	__aligned_u64 user_compl_ctx;

kernel-headers/rdma/rdma_user_cm.h

@@ -68,7 +68,8 @@ enum {
 	RDMA_USER_CM_CMD_BIND,
 	RDMA_USER_CM_CMD_RESOLVE_ADDR,
 	RDMA_USER_CM_CMD_JOIN_MCAST,
-	RDMA_USER_CM_CMD_RESOLVE_IB_SERVICE
+	RDMA_USER_CM_CMD_RESOLVE_IB_SERVICE,
+	RDMA_USER_CM_CMD_WRITE_CM_EVENT,
 };
 
 /* See IBTA Annex A11, servies ID bytes 4 & 5 */
@@ -304,6 +305,7 @@ struct rdma_ucm_event_resp {
 	union {
 		struct rdma_ucm_conn_param conn;
 		struct rdma_ucm_ud_param   ud;
+		__u32 arg32[2];
 	} param;
 	__u32 reserved;
 	struct rdma_ucm_ece ece;
@@ -362,4 +364,16 @@ struct rdma_ucm_resolve_ib_service {
 	__u32 id;
 	struct rdma_ucm_ib_service ibs;
 };
+
+struct rdma_ucm_write_cm_event {
+	__u32 id;
+	__u32 reserved;
+	__u32 event;
+	__u32 status;
+	union {
+		struct rdma_ucm_conn_param conn;
+		struct rdma_ucm_ud_param ud;
+		__u64 arg;
+	} param;
+};
 #endif /* RDMA_USER_CM_H */

providers/bnxt_re/CMakeLists.txt

@@ -1,6 +1,19 @@
-rdma_provider(bnxt_re
+rdma_shared_provider(bnxt_re bnxt_re.map
+	1 1.1.${PACKAGE_VERSION}
 	db.c
 	main.c
 	memory.c
 	verbs.c
+	dv.c
 )
+
+publish_headers(infiniband
+  bnxt_re_dv.h
+)
+
+rdma_pkg_config("bnxt_re" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
+
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+  target_include_directories(bnxt_re PUBLIC ".")
+  target_link_libraries(bnxt_re LINK_PRIVATE LTTng::UST)
+endif()

providers/bnxt_re/bnxt_re.map

@@ -0,0 +1,17 @@
+/* Export symbols should be added below according to
+   Documentation/versioning.md document. */
+BNXT_RE_1.1 {
+	global:
+		bnxt_re_dv_alloc_db_region;
+		bnxt_re_dv_free_db_region;
+		bnxt_re_dv_umem_reg;
+		bnxt_re_dv_umem_dereg;
+		bnxt_re_dv_get_default_db_region;
+		bnxt_re_dv_create_cq;
+		bnxt_re_dv_destroy_cq;
+		bnxt_re_dv_create_qp;
+		bnxt_re_dv_destroy_qp;
+		bnxt_re_dv_modify_qp;
+		bnxt_re_dv_query_qp;
+	local: *;
+};

providers/bnxt_re/bnxt_re_dv.h

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2025, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Direct verb support user interface header
+ */
+
+#ifndef __BNXT_RE_DV_H__
+#define __BNXT_RE_DV_H__
+
+#include <stdint.h>
+#include <infiniband/verbs.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct bnxt_re_dv_db_region_attr {
+	uint32_t handle;
+	uint32_t dpi;
+	uint64_t umdbr;
+	__u64 *dbr;
+};
+
+enum  bnxt_re_dv_umem_in_flags {
+	BNXT_RE_DV_UMEM_FLAGS_DMABUF = 1 << 0,
+};
+
+struct bnxt_re_dv_umem_reg_attr {
+	void *addr;
+	size_t size;
+	uint32_t access_flags;
+	uint64_t pgsz_bitmap;
+	uint64_t comp_mask;
+	int dmabuf_fd;
+};
+
+struct bnxt_re_dv_cq_init_attr {
+	uint64_t cq_handle;
+	void *umem_handle; /* umem_handle from umem_reg */
+	uint64_t cq_umem_offset;	/* offset into umem */
+	uint32_t ncqe;
+};
+
+struct bnxt_re_dv_cq_attr {
+	uint32_t ncqe; /* no. of entries */
+	uint32_t cqe_size; /* size of entries */
+};
+
+struct bnxt_re_dv_qp_init_attr {
+	/* Standard ibv params */
+	enum ibv_qp_type qp_type;
+	uint32_t max_send_wr;
+	uint32_t max_recv_wr;
+	uint32_t max_send_sge;
+	uint32_t max_recv_sge;
+	uint32_t max_inline_data;
+	struct ibv_cq *send_cq;
+	struct ibv_cq *recv_cq;
+	struct ibv_srq *srq;
+
+	/* DV params */
+	uint64_t qp_handle;	/* to match with cqe */
+	void *dbr_handle;	/* dbr_handle from alloc_dbr */
+	void *sq_umem_handle;	/* umem_handle from umem_reg */
+	uint64_t sq_umem_offset;	/* offset into umem */
+	uint32_t sq_len;	/* sq length including MSN area */
+	uint32_t sq_slots;	/* sq length in slots */
+	void *rq_umem_handle;	/* umem_handle from umem_reg */
+	uint64_t rq_umem_offset;	/* offset into umem */
+	uint32_t sq_wqe_sz;     /* sq wqe size */
+	uint32_t sq_psn_sz;     /* sq psn size */
+	uint32_t sq_npsn;       /* sq num psn entries */
+	uint32_t rq_len;	/* rq length */
+	uint32_t rq_slots;	/* rq length in slots */
+	uint32_t rq_wqe_sz;     /* rq wqe size */
+	uint64_t comp_mask;	/* compatibility mask for future updates */
+};
+
+struct bnxt_re_dv_db_region_attr *
+bnxt_re_dv_alloc_db_region(struct ibv_context *ctx);
+int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
+			      struct bnxt_re_dv_db_region_attr *attr);
+int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
+				     struct bnxt_re_dv_db_region_attr *out);
+void *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx,
+			  struct bnxt_re_dv_umem_reg_attr *in);
+int bnxt_re_dv_umem_dereg(void *umem_handle);
+struct ibv_cq *bnxt_re_dv_create_cq(struct ibv_context *ibvctx,
+				    struct bnxt_re_dv_cq_init_attr *cq_attr);
+int bnxt_re_dv_destroy_cq(struct ibv_cq *ibv_cq);
+struct ibv_qp *bnxt_re_dv_create_qp(struct ibv_pd *pd,
+				    struct bnxt_re_dv_qp_init_attr *qp_attr);
+int bnxt_re_dv_destroy_qp(struct ibv_qp *ibvqp);
+int bnxt_re_dv_modify_qp(struct ibv_qp *ibv_qp, struct ibv_qp_attr *attr,
+			 int attr_mask);
+int bnxt_re_dv_query_qp(void *qp_handle, struct ib_uverbs_qp_attr *attr);
+#ifdef __cplusplus
+}
+#endif
+#endif /* __BNXT_RE_DV_H__ */