Skip to content

Commit a47345d

Browse files
ztefuyanztefuyan
committed
libzrdma: Add support for cq
This patch add the following verbs: create_cq destroy_cq modify_cq poll_cq req_notify_cq resize_cq cq_event also add other information: debug interface function private interface for operation hardware on-behalf-of: @ZTE li.fuyan@zte.com.cn Signed-off-by: zte_lifuyan <li.fuyan@zte.com.cn>
1 parent 9017116 commit a47345d

File tree

13 files changed

+3557
-11
lines changed

13 files changed

+3557
-11
lines changed

‎kernel-headers/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ publish_internal_headers(rdma
2727
rdma/siw-abi.h
2828
rdma/vmw_pvrdma-abi.h
2929
rdma/zxdh-abi.h
30+
rdma/zxdh_user_ioctl_verbs.h
3031
)
3132

3233
publish_internal_headers(rdma/hfi

‎kernel-headers/rdma/zxdh_user_ioctl_cmds.h‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
2+
/*
3+
* Copyright (c) 2024 ZTE Corporation.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* OpenIB.org BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or
12+
* without modification, are permitted provided that the following
13+
* conditions are met:
14+
*
15+
* - Redistributions of source code must retain the above
16+
* copyright notice, this list of conditions and the following
17+
* disclaimer.
18+
*
19+
* - Redistributions in binary form must reproduce the above
20+
* copyright notice, this list of conditions and the following
21+
* disclaimer in the documentation and/or other materials
22+
* provided with the distribution.
23+
*
24+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
29+
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31+
* SOFTWARE.
32+
*/
33+
34+
#ifndef ZXDH_USER_IOCTL_CMDS_H
35+
#define ZXDH_USER_IOCTL_CMDS_H
36+
37+
#include <linux/types.h>
38+
#include <rdma/ib_user_ioctl_cmds.h>
39+
40+
enum zxdh_ib_dev_get_log_trace_attrs {
41+
ZXDH_IB_ATTR_DEV_GET_LOG_TARCE_SWITCH = (1U << UVERBS_ID_NS_SHIFT),
42+
};
43+
44+
enum zxdh_ib_dev_set_log_trace_attrs {
45+
ZXDH_IB_ATTR_DEV_SET_LOG_TARCE_SWITCH = (1U << UVERBS_ID_NS_SHIFT),
46+
};
47+
48+
enum zxdh_ib_dev_methods {
49+
ZXDH_IB_METHOD_DEV_GET_LOG_TRACE = (1U << UVERBS_ID_NS_SHIFT),
50+
ZXDH_IB_METHOD_DEV_SET_LOG_TRACE,
51+
};
52+
53+
enum zxdh_ib_qp_modify_udp_sport_attrs {
54+
ZXDH_IB_ATTR_QP_UDP_PORT = (1U << UVERBS_ID_NS_SHIFT),
55+
ZXDH_IB_ATTR_QP_QPN,
56+
};
57+
58+
enum zxdh_ib_qp_query_qpc_attrs {
59+
ZXDH_IB_ATTR_QP_QUERY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
60+
ZXDH_IB_ATTR_QP_QUERY_RESP,
61+
};
62+
63+
enum zxdh_ib_qp_modify_qpc_attrs {
64+
ZXDH_IB_ATTR_QP_MODIFY_QPC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
65+
ZXDH_IB_ATTR_QP_MODIFY_QPC_REQ,
66+
ZXDH_IB_ATTR_QP_MODIFY_QPC_MASK,
67+
};
68+
69+
enum zxdh_ib_qp_reset_qp_attrs {
70+
ZXDH_IB_ATTR_QP_RESET_QP_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
71+
ZXDH_IB_ATTR_QP_RESET_OP_CODE,
72+
};
73+
74+
enum zxdh_ib_qp_methods {
75+
ZXDH_IB_METHOD_QP_MODIFY_UDP_SPORT = (1U << UVERBS_ID_NS_SHIFT),
76+
ZXDH_IB_METHOD_QP_QUERY_QPC,
77+
ZXDH_IB_METHOD_QP_MODIFY_QPC,
78+
ZXDH_IB_METHOD_QP_RESET_QP,
79+
};
80+
81+
enum zxdh_ib_objects {
82+
ZXDH_IB_OBJECT_DEV = (1U << UVERBS_ID_NS_SHIFT),
83+
ZXDH_IB_OBJECT_QP_OBJ,
84+
ZXDH_IB_OBJECT_DEVICE_EX,
85+
};
86+
87+
#endif

‎kernel-headers/rdma/zxdh_user_ioctl_verbs.h‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
2+
/*
3+
* Copyright (c) 2024 ZTE Corporation.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* OpenIB.org BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or
12+
* without modification, are permitted provided that the following
13+
* conditions are met:
14+
*
15+
* - Redistributions of source code must retain the above
16+
* copyright notice, this list of conditions and the following
17+
* disclaimer.
18+
*
19+
* - Redistributions in binary form must reproduce the above
20+
* copyright notice, this list of conditions and the following
21+
* disclaimer in the documentation and/or other materials
22+
* provided with the distribution.
23+
*
24+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
29+
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31+
* SOFTWARE.
32+
*/
33+
#ifndef ZXDH_USER_IOCTL_VERBS_H
34+
#define ZXDH_USER_IOCTL_VERBS_H
35+
36+
#include <linux/types.h>
37+
#include <stdbool.h>
38+
39+
//todo ailgn
40+
struct zxdh_query_qpc_resp {
41+
__u8 retry_flag;
42+
__u8 rnr_retry_flag;
43+
__u8 read_retry_flag;
44+
__u8 cur_retry_count;
45+
__u8 retry_cqe_sq_opcode;
46+
__u8 err_flag;
47+
__u8 ack_err_flag;
48+
__u8 package_err_flag;
49+
__u8 recv_err_flag;
50+
__u8 retry_count;
51+
__u32 tx_last_ack_psn;
52+
};
53+
54+
struct zxdh_modify_qpc_req {
55+
__u8 retry_flag;
56+
__u8 rnr_retry_flag;
57+
__u8 read_retry_flag;
58+
__u8 cur_retry_count;
59+
__u8 retry_cqe_sq_opcode;
60+
__u8 err_flag;
61+
__u8 ack_err_flag;
62+
__u8 package_err_flag;
63+
};
64+
65+
#endif

‎providers/zrdma/CMakeLists.txt‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,27 @@
1+
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2+
# Copyright (c) 2024 ZTE Corporation.
3+
14
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
2-
rdma_shared_provider(zrdma libzrdma.map
3-
1 1.1.${PACKAGE_VERSION}
5+
rdma_provider(zrdma
6+
zxdh_hw.c
47
zxdh_zrdma.c
58
zxdh_verbs.c
9+
private_verbs_cmd.c
610
)
711

8-
rdma_pkg_config("zrdma" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
12+
include_directories(${NL_INCLUDE_DIRS})
13+
14+
publish_headers(infiniband
15+
zxdh_dv.h
16+
zxdh_devids.h
17+
)
18+
19+
if(TARGET zrdma)
20+
target_link_libraries(zrdma PRIVATE nl-3)
21+
rdma_pkg_config("zrdma" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
22+
endif()
23+
if(TARGET zrdma-rdmav${IBVERBS_PABI_VERSION})
24+
target_link_libraries(zrdma-rdmav${IBVERBS_PABI_VERSION} PRIVATE nl-3)
25+
rdma_pkg_config("zrdma-rdmav${IBVERBS_PABI_VERSION}" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
26+
endif()
27+

‎providers/zrdma/private_verbs_cmd.c‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2+
/*
3+
* Copyright (c) 2024 ZTE Corporation.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* OpenIB.org BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or
12+
* without modification, are permitted provided that the following
13+
* conditions are met:
14+
*
15+
* - Redistributions of source code must retain the above
16+
* copyright notice, this list of conditions and the following
17+
* disclaimer.
18+
*
19+
* - Redistributions in binary form must reproduce the above
20+
* copyright notice, this list of conditions and the following
21+
* disclaimer in the documentation and/or other materials
22+
* provided with the distribution.
23+
*
24+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
29+
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31+
* SOFTWARE.
32+
*/
33+
#include <rdma/zxdh_user_ioctl_cmds.h>
34+
#include <rdma/zxdh_user_ioctl_verbs.h>
35+
#include "private_verbs_cmd.h"
36+
#include "zxdh_dv.h"
37+
38+
static inline uint32_t get_log2(uint32_t x)
39+
{
40+
uint32_t r = 0;
41+
42+
for (x >>= 1; x > 0; x >>= 1)
43+
r++;
44+
return r;
45+
}
46+
47+
static void copy_query_qpc(struct zxdh_query_qpc_resp *resp,
48+
struct zxdh_rdma_qpc *qpc)
49+
{
50+
qpc->ack_err_flag = resp->ack_err_flag;
51+
qpc->retry_flag = resp->retry_flag;
52+
qpc->rnr_retry_flag = resp->rnr_retry_flag;
53+
qpc->cur_retry_count = resp->cur_retry_count;
54+
qpc->retry_cqe_sq_opcode = resp->retry_cqe_sq_opcode;
55+
qpc->err_flag = resp->err_flag;
56+
qpc->package_err_flag = resp->package_err_flag;
57+
qpc->recv_err_flag = resp->recv_err_flag;
58+
qpc->tx_last_ack_psn = resp->tx_last_ack_psn;
59+
qpc->retry_count = resp->retry_count;
60+
qpc->read_retry_flag = resp->read_retry_flag;
61+
}
62+
63+
static int _zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc)
64+
{
65+
DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
66+
ZXDH_IB_METHOD_QP_QUERY_QPC, 2);
67+
int ret;
68+
struct zxdh_query_qpc_resp resp_ex = { 0 };
69+
70+
fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_QUERY_HANDLE, qp->handle);
71+
fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_QP_QUERY_RESP, &resp_ex);
72+
73+
ret = execute_ioctl(qp->context, cmd);
74+
if (ret)
75+
return ret;
76+
77+
copy_query_qpc(&resp_ex, qpc);
78+
return 0;
79+
}
80+
81+
static struct zxdh_uvcontext_ops zxdh_ctx_ops = {
82+
.query_qpc = _zxdh_query_qpc,
83+
};
84+
85+
static inline struct zxdh_uvcontext *to_zxdhtx(struct ibv_context *ibctx)
86+
{
87+
return container_of(ibctx, struct zxdh_uvcontext, ibv_ctx.context);
88+
}
89+
90+
int zxdh_reset_qp(struct ibv_qp *qp, uint64_t opcode)
91+
{
92+
struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops;
93+
94+
if (!dvops || !dvops->reset_qp)
95+
return -EOPNOTSUPP;
96+
return dvops->reset_qp(qp, opcode);
97+
}
98+
99+
int zxdh_modify_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc,
100+
uint64_t qpc_mask)
101+
{
102+
struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops;
103+
104+
if (!dvops || !dvops->modify_qpc)
105+
return -EOPNOTSUPP;
106+
return dvops->modify_qpc(qp, qpc, qpc_mask);
107+
}
108+
109+
int zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc)
110+
{
111+
struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops;
112+
113+
if (!dvops || !dvops->query_qpc)
114+
return -EOPNOTSUPP;
115+
116+
return dvops->query_qpc(qp, qpc);
117+
}
118+
119+
void add_private_ops(struct zxdh_uvcontext *iwvctx)
120+
{
121+
iwvctx->cxt_ops = &zxdh_ctx_ops;
122+
}

‎providers/zrdma/private_verbs_cmd.h‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
2+
/*
3+
* Copyright (c) 2024 ZTE Corporation.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* OpenIB.org BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or
12+
* without modification, are permitted provided that the following
13+
* conditions are met:
14+
*
15+
* - Redistributions of source code must retain the above
16+
* copyright notice, this list of conditions and the following
17+
* disclaimer.
18+
*
19+
* - Redistributions in binary form must reproduce the above
20+
* copyright notice, this list of conditions and the following
21+
* disclaimer in the documentation and/or other materials
22+
* provided with the distribution.
23+
*
24+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
29+
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31+
* SOFTWARE.
32+
*/
33+
#ifndef ZXDH_RDMA_PRIVATE_VERBS_CMD_H
34+
#define ZXDH_RDMA_PRIVATE_VERBS_CMD_H
35+
36+
#include "zxdh_zrdma.h"
37+
#include "zxdh_dv.h"
38+
39+
enum zxdh_rdma_tool_flags {
40+
ZXDH_QP_EXTEND_OP = 1 << 0,
41+
ZXDH_CAPTURE = 1 << 1,
42+
ZXDH_GET_HW_DATA = 1 << 2,
43+
ZXDH_GET_HW_OBJECT_DATA = 1 << 3,
44+
ZXDH_CHECK_HW_HEALTH = 1 << 4,
45+
ZXDH_RDMA_TOOL_CFG_DEV_PARAM = 1 << 5,
46+
ZXDH_RDMA_TOOL_SHOW_RES_MAP = 1 << 5,
47+
};
48+
49+
struct zxdh_uvcontext_ops {
50+
int (*modify_qp_udp_sport)(struct ibv_context *ibctx,
51+
uint16_t udp_sport, uint32_t qpn);
52+
int (*set_log_trace_switch)(struct ibv_context *ibctx,
53+
uint8_t switch_status);
54+
int (*get_log_trace_switch)(struct ibv_context *ibctx,
55+
uint8_t *switch_status);
56+
int (*query_qpc)(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc);
57+
int (*modify_qpc)(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc,
58+
uint64_t qpc_mask);
59+
int (*reset_qp)(struct ibv_qp *qp, uint64_t opcode);
60+
};
61+
62+
void add_private_ops(struct zxdh_uvcontext *iwvctx);
63+
64+
#endif

0 commit comments

Comments
 (0)