From 84cf060aca0c3aff2bbfd5862231f94fb9e4fa94 Mon Sep 17 00:00:00 2001 From: yonatanc Date: Sun, 12 Mar 2017 10:52:22 +0200 Subject: [PATCH 1/4] fix compilation ptr_idr --- src/cmd.c | 10 +++++----- src/ibverbs.h | 20 ++++++++++---------- src/verbs.c | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/cmd.c b/src/cmd.c index 75e066b..8d73eaf 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -313,7 +313,7 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - pd->handle = cmd.attrs[0].ptr_idr; + pd->handle = cmd.attrs[0].data; pd->context = context; return 0; @@ -433,7 +433,7 @@ int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - mr->handle = cmd.attrs[0].ptr_idr; + mr->handle = cmd.attrs[0].data; mr->lkey = resp.lkey; mr->rkey = resp.rkey; mr->context = pd->context; @@ -577,7 +577,7 @@ int ibv_cmd_create_cq(struct ibv_context *context, int cqe, printf("%s:%d\n", __func__, __LINE__); cq->context = context; cq->cqe = _cqe_out; - cq->handle = cmd.attrs[0].ptr_idr; + cq->handle = cmd.attrs[0].data; printf("%s:%d\n", __func__, __LINE__); return 0; @@ -1102,7 +1102,7 @@ int ibv_cmd_create_qp_ex2(struct ibv_context *context, create_qp_ioctl_handle_resp_common(context, qp, qp_attr, &resp, - cmd.attrs[0].ptr_idr, + cmd.attrs[0].data, vxrcd, vqp_sz); @@ -1152,7 +1152,7 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context, create_qp_ioctl_handle_resp_common(context, qp, attr_ex, &resp, - cmd.attrs[0].ptr_idr, + cmd.attrs[0].data, vxrcd, vqp_sz); diff --git a/src/ibverbs.h b/src/ibverbs.h index cd8dc64..855a79b 100644 --- a/src/ibverbs.h +++ b/src/ibverbs.h @@ -143,14 +143,14 @@ static inline void fill_ioctl_hdr(struct ib_uverbs_ioctl_hdr *cmd, { cmd->length = length; cmd->flags = 0; - cmd->driver_id = 0; + cmd->reserved = 0; cmd->object_type = object_type; cmd->action = action; cmd->num_attrs = num_attr; } static inline void fill_attr(struct ib_uverbs_attr *attr, uint16_t attr_id, - uint16_t len, void *ptr_idr) + uint16_t len, void *data) { attr->attr_id = attr_id; attr->len = len; @@ -159,20 +159,20 @@ static inline void fill_attr(struct ib_uverbs_attr *attr, uint16_t attr_id, } static inline void fill_attr_in(struct ib_uverbs_attr *attr, uint16_t attr_id, - uint16_t len, void *ptr_idr) + uint16_t len, void *data) { - fill_attr(attr, attr_id, len, ptr_idr); + fill_attr(attr, attr_id, len, data); if (len <= sizeof(uint64_t)) - memcpy((void *)&attr->ptr_idr, ptr_idr, len); + memcpy((void *)&attr->data, data, len); else - attr->ptr_idr = (uint64_t)ptr_idr; + attr->data = (uint64_t)data; } static inline void fill_attr_out(struct ib_uverbs_attr *attr, uint16_t attr_id, - uint16_t len, void *ptr_idr) + uint16_t len, void *data) { - fill_attr(attr, attr_id, len, ptr_idr); - attr->ptr_idr = (uint64_t)ptr_idr; + fill_attr(attr, attr_id, len, data); + attr->data = (uint64_t)data; } static inline void fill_attr_obj(struct ib_uverbs_attr *attr, uint16_t attr_id, @@ -181,6 +181,6 @@ static inline void fill_attr_obj(struct ib_uverbs_attr *attr, uint16_t attr_id, attr->attr_id = attr_id; attr->len = 0; attr->reserved = 0; - attr->ptr_idr = idr; + attr->data = idr; } #endif /* IB_VERBS_H */ diff --git a/src/verbs.c b/src/verbs.c index 1ff9633..0294217 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -349,7 +349,7 @@ struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context) (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); channel->context = context; - channel->fd = cmd.attrs.ptr_idr; + channel->fd = cmd.attrs.data; channel->refcnt = 0; return channel; From 5deedc55390ab806b51feddaa79dee7655fede5e Mon Sep 17 00:00:00 2001 From: yonatanc Date: Wed, 8 Mar 2017 18:01:21 +0200 Subject: [PATCH 2/4] Add query_qp verb --- include/infiniband/kern-abi.h | 12 +++++++++++- src/cmd.c | 28 ++++++++++++++++++++-------- src/verbs.c | 2 +- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h index e787dac..e57979c 100644 --- a/include/infiniband/kern-abi.h +++ b/include/infiniband/kern-abi.h @@ -1202,6 +1202,15 @@ enum uverbs_destroy_qp_cmd_attr { DESTROY_QP_RESERVED }; +// this enum should also reside in +// kernel/include/rdma/uverbs_std_types.h +enum uverbs_query_qp_cmd_attr_ids { + QUERY_QP_HANDLE, + QUERY_QP_ATTR_MASK, + QUERY_QP_RESP, + QUERY_QP_RESERVED +}; + enum uverbs_create_cq_cmd_attr { CREATE_CQ_HANDLE, CREATE_CQ_CQE, @@ -1320,7 +1329,8 @@ enum uverbs_actions_qp_ops { UVERBS_QP_CREATE, UVERBS_QP_CREATE_XRC_TGT, UVERBS_QP_MODIFY, - UVERBS_QP_DESTROY + UVERBS_QP_DESTROY, + UVERBS_QP_QUERY, }; enum uverbs_actions_pd_ops { diff --git a/src/cmd.c b/src/cmd.c index 8d73eaf..7df0197 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -76,7 +76,7 @@ int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *leg attr - cmd.attrs); ret = ioctl(context->cmd_fd, RDMA_VERBS_IOCTL, &cmd); - printf("ret is %d\n", ret); + printf("ret is %lu\n", ret); if (ret) return errno; @@ -1267,21 +1267,32 @@ int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, return 0; } -int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, +struct ibv_ioctl_cmd_query_qp { + struct ib_uverbs_ioctl_hdr hdr; + struct ib_uverbs_attr attrs[QUERY_QP_RESERVED]; +} __attribute__((packed, aligned(4))); + +int ibv_cmd_query_qp(struct ibv_qp *qp, + struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr, struct ibv_query_qp *cmd, size_t cmd_size) { + struct ibv_ioctl_cmd_query_qp iocmd; + struct ib_uverbs_attr *cattr = iocmd.attrs; struct ibv_query_qp_resp resp; + int ret; - IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp); - cmd->qp_handle = qp->handle; - cmd->attr_mask = attr_mask; + fill_attr_obj(cattr++, QUERY_QP_HANDLE, qp->handle); + fill_attr_in(cattr++, QUERY_QP_ATTR_MASK, sizeof(__u32), &attr_mask); + fill_attr_out(cattr++, QUERY_QP_RESP, sizeof(resp), &resp); - if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; + fill_ioctl_hdr(&iocmd.hdr, UVERBS_TYPE_QP, (void *)cattr - (void *)&iocmd, + UVERBS_QP_QUERY, cattr - iocmd.attrs); - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); + ret = ioctl(qp->context->cmd_fd, RDMA_VERBS_IOCTL, &iocmd); + if (ret) + return errno; attr->qkey = resp.qkey; attr->rq_psn = resp.rq_psn; @@ -1347,6 +1358,7 @@ int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, init_attr->sq_sig_all = resp.sq_sig_all; return 0; + } struct ibv_ioctl_cmd_modify_qp { diff --git a/src/verbs.c b/src/verbs.c index 0294217..d796fa2 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -344,7 +344,7 @@ struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context) UVERBS_COMP_CHANNEL_CREATE, CREATE_COMP_CHANNEL_RESERVED); if (ioctl(context->cmd_fd, RDMA_VERBS_IOCTL, &cmd)) - return errno; + return NULL; (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); From d6e9d0a9619fd8442d9fb178739bfb65669b79b4 Mon Sep 17 00:00:00 2001 From: yonatanc Date: Sun, 12 Mar 2017 13:25:01 +0200 Subject: [PATCH 3/4] Add rereg_mr verb --- include/infiniband/kern-abi.h | 9 ++++++ src/cmd.c | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h index e57979c..a1a458c 100644 --- a/include/infiniband/kern-abi.h +++ b/include/infiniband/kern-abi.h @@ -1211,6 +1211,14 @@ enum uverbs_query_qp_cmd_attr_ids { QUERY_QP_RESERVED }; +enum uverbs_rereg_mr_cmd_attr_ids { + REREG_MR_HANDLE, + REREG_MR_PD_HANDLE, + REREG_MR_CMD, + REREG_MR_RESP, + REREG_MR_RESERVED, +}; + enum uverbs_create_cq_cmd_attr { CREATE_CQ_HANDLE, CREATE_CQ_CQE, @@ -1314,6 +1322,7 @@ enum uverbs_dereg_mr { enum uverbs_actions_mr_ops { UVERBS_MR_REG, UVERBS_MR_DEREG, + UVERBS_MR_REREG, }; enum uverbs_actions_comp_channel_ops { diff --git a/src/cmd.c b/src/cmd.c index 7df0197..04e7b1c 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -441,6 +441,64 @@ int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, return 0; } +#define new_rereg_me +#ifdef new_rereg_me +struct ibv_ioctl_rereg_mr_cmd { + __u64 response; + __u32 mr_handle; + __u32 flags; + __u64 start; + __u64 length; + __u64 hca_va; + __u32 pd_handle; + __u32 access_flags; +}; + +struct ibv_ioctl_cmd_rereg_mr { + struct ib_uverbs_ioctl_hdr hdr; + struct ib_uverbs_attr attrs[REREG_MR_RESERVED]; +}__attribute__((packed, aligned(4))); + +int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, + size_t length, uint64_t hca_va, int access, + struct ibv_pd *pd, struct ibv_rereg_mr *cmd, + size_t cmd_sz, struct ibv_rereg_mr_resp *resp, + size_t resp_sz) +{ + struct ibv_ioctl_cmd_rereg_mr iocmd; + struct ib_uverbs_attr *cattr = iocmd.attrs; + struct ibv_ioctl_rereg_mr_cmd rereg_mr_cmd; + + fill_attr_obj(cattr++, REREG_MR_HANDLE, mr->handle); + + if (flags & IBV_REREG_MR_CHANGE_PD) + fill_attr_obj(cattr++, REREG_MR_PD_HANDLE, pd->handle); + else + fill_attr_obj(cattr++, REREG_MR_PD_HANDLE, 0); + + fill_attr_in(cattr++, REREG_MR_CMD, sizeof(struct ibv_ioctl_rereg_mr_cmd), &rereg_mr_cmd); + fill_attr_out(cattr++, REREG_MR_RESP, sizeof(struct ibv_rereg_mr_resp), resp); + + rereg_mr_cmd.start = (uintptr_t)addr; + rereg_mr_cmd.length = length; + rereg_mr_cmd.hca_va = hca_va; + rereg_mr_cmd.flags = flags; + rereg_mr_cmd.access_flags = access; + + fill_ioctl_hdr(&iocmd.hdr, UVERBS_TYPE_MR, (void *)cattr - (void *)&iocmd, + UVERBS_MR_REREG, cattr - iocmd.attrs); + + if (ioctl(mr->context->cmd_fd, RDMA_VERBS_IOCTL, &iocmd)) + return errno; + + mr->lkey = resp->lkey; + mr->rkey = resp->rkey; + if (flags & IBV_REREG_MR_CHANGE_PD) + mr->context = pd->context; + + return 0; +} +#else int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, size_t length, uint64_t hca_va, int access, struct ibv_pd *pd, struct ibv_rereg_mr *cmd, @@ -469,6 +527,7 @@ int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, return 0; } +#endif struct ibv_ioctl_cmd_dereg_mr { struct ib_uverbs_ioctl_hdr hdr; From b1779220e7b9039e9199fada6d0d88e90173b689 Mon Sep 17 00:00:00 2001 From: yonatanc Date: Wed, 15 Mar 2017 10:54:34 +0200 Subject: [PATCH 4/4] Merge user kernel headers rdma_user_ioctl_cmd.h will replace kern-abi.h to be included from libibverbs to share common definitions of ioctl commands. --- include/infiniband/kern-abi.h | 186 +--------------------------------- src/cmd.c | 2 +- 2 files changed, 2 insertions(+), 186 deletions(-) diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h index a1a458c..168d047 100644 --- a/include/infiniband/kern-abi.h +++ b/include/infiniband/kern-abi.h @@ -36,6 +36,7 @@ #define KERN_ABI_H #include +#include /* * This file must be kept in sync with the kernel's version of @@ -1168,191 +1169,6 @@ enum { #define UVERBS_UHW_SZ ((UVERBS_UHW_OUT & ~IB_UVERBS_SCOPE_DRIVER) + 1) -enum uverbs_common_types { - UVERBS_TYPE_DEVICE, /* Don't use IDRs here */ - UVERBS_TYPE_PD, - UVERBS_TYPE_COMP_CHANNEL, - UVERBS_TYPE_CQ, - UVERBS_TYPE_QP, - UVERBS_TYPE_SRQ, - UVERBS_TYPE_AH, - UVERBS_TYPE_MR, - UVERBS_TYPE_MW, - UVERBS_TYPE_FLOW, - UVERBS_TYPE_XRCD, - UVERBS_TYPE_LAST, -}; - -enum uverbs_create_qp_cmd_attr { - CREATE_QP_HANDLE, - CREATE_QP_PD_HANDLE, - CREATE_QP_SEND_CQ, - CREATE_QP_RECV_CQ, - CREATE_QP_SRQ, - CREATE_QP_USER_HANDLE, - CREATE_QP_CMD, - CREATE_QP_CMD_FLAGS, - CREATE_QP_RESP, - CREATE_QP_RESERVED -}; - -enum uverbs_destroy_qp_cmd_attr { - DESTROY_QP_HANDLE, - DESTROY_QP_EVENTS_REPORTED, - DESTROY_QP_RESERVED -}; - -// this enum should also reside in -// kernel/include/rdma/uverbs_std_types.h -enum uverbs_query_qp_cmd_attr_ids { - QUERY_QP_HANDLE, - QUERY_QP_ATTR_MASK, - QUERY_QP_RESP, - QUERY_QP_RESERVED -}; - -enum uverbs_rereg_mr_cmd_attr_ids { - REREG_MR_HANDLE, - REREG_MR_PD_HANDLE, - REREG_MR_CMD, - REREG_MR_RESP, - REREG_MR_RESERVED, -}; - -enum uverbs_create_cq_cmd_attr { - CREATE_CQ_HANDLE, - CREATE_CQ_CQE, - CREATE_CQ_USER_HANDLE, - CREATE_CQ_COMP_CHANNEL, - CREATE_CQ_COMP_VECTOR, - CREATE_CQ_FLAGS, - CREATE_CQ_RESP_CQE, - CREATE_CQ_RESERVED, -}; - -enum uverbs_destroy_cq_cmd_attr { - DESTROY_CQ_HANDLE, - DESTROY_CQ_RESP, - DESTROY_CQ_RESERVED -}; - -enum uverbs_create_qp_xrc_tgt_cmd_attr { - CREATE_QP_XRC_TGT_HANDLE, - CREATE_QP_XRC_TGT_XRCD, - CREATE_QP_XRC_TGT_USER_HANDLE, - CREATE_QP_XRC_TGT_CMD, - CREATE_QP_XRC_TGT_CMD_FLAGS, - CREATE_QP_XRC_TGT_RESP, - CREATE_QP_XRC_TGT_RESERVED -}; - -enum uverbs_modify_qp_cmd_attr { - MODIFY_QP_HANDLE, - MODIFY_QP_STATE, - MODIFY_QP_CUR_STATE, - MODIFY_QP_EN_SQD_ASYNC_NOTIFY, - MODIFY_QP_ACCESS_FLAGS, - MODIFY_QP_PKEY_INDEX, - MODIFY_QP_PORT, - MODIFY_QP_QKEY, - MODIFY_QP_AV, - MODIFY_QP_PATH_MTU, - MODIFY_QP_TIMEOUT, - MODIFY_QP_RETRY_CNT, - MODIFY_QP_RNR_RETRY, - MODIFY_QP_RQ_PSN, - MODIFY_QP_MAX_RD_ATOMIC, - MODIFY_QP_ALT_PATH, - MODIFY_QP_MIN_RNR_TIMER, - MODIFY_QP_SQ_PSN, - MODIFY_QP_MAX_DEST_RD_ATOMIC, - MODIFY_QP_PATH_MIG_STATE, - MODIFY_QP_DEST_QPN, - MODIFY_QP_RESERVED -}; - -enum uverbs_create_comp_channel_cmd_attr { - CREATE_COMP_CHANNEL_FD, - CREATE_COMP_CHANNEL_RESERVED -}; - -enum uverbs_get_context { - GET_CONTEXT_RESP, - GET_CONTEXT_RESERVED, -}; - -enum uverbs_query_device { - QUERY_DEVICE_RESP, - QUERY_DEVICE_ODP, - QUERY_DEVICE_TIMESTAMP_MASK, - QUERY_DEVICE_HCA_CORE_CLOCK, - QUERY_DEVICE_CAP_FLAGS, - QUERY_DEVICE_CAP_RESERVED, -}; - -enum uverbs_query_port { - QUERY_PORT_PORT_NUM, - QUERY_PORT_RESP, - QUERY_PORT_RESERVED -}; - -enum uverbs_alloc_pd { - ALLOC_PD_HANDLE, - ALLOC_PD_RESERVED, -}; - -enum uverbs_dealloc_pd { - DEALLOC_PD_HANDLE, - DEALLOC_PD_RESERVED, -}; - -enum uverbs_reg_mr { - REG_MR_HANDLE, - REG_MR_PD_HANDLE, - REG_MR_CMD, - REG_MR_RESP, - REG_MR_RESERVED, -}; - -enum uverbs_dereg_mr { - DEREG_MR_HANDLE, - DEREG_MR_RESERVED -}; - -enum uverbs_actions_mr_ops { - UVERBS_MR_REG, - UVERBS_MR_DEREG, - UVERBS_MR_REREG, -}; - -enum uverbs_actions_comp_channel_ops { - UVERBS_COMP_CHANNEL_CREATE, -}; - -enum uverbs_actions_cq_ops { - UVERBS_CQ_CREATE, - UVERBS_CQ_DESTROY, -}; - -enum uverbs_actions_qp_ops { - UVERBS_QP_CREATE, - UVERBS_QP_CREATE_XRC_TGT, - UVERBS_QP_MODIFY, - UVERBS_QP_DESTROY, - UVERBS_QP_QUERY, -}; - -enum uverbs_actions_pd_ops { - UVERBS_PD_ALLOC, - UVERBS_PD_DEALLOC, -}; - -enum uverbs_actions_device_ops { - UVERBS_DEVICE_ALLOC_CONTEXT, - UVERBS_DEVICE_QUERY, - UVERBS_DEVICE_PORT_QUERY, -}; - struct ib_uverbs_ioctl_create_qp { __u32 max_send_wr; __u32 max_recv_wr; diff --git a/src/cmd.c b/src/cmd.c index 04e7b1c..8f29337 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -163,7 +163,7 @@ int ibv_cmd_query_device(struct ibv_context *context, struct ibv_ioctl_cmd_query_device_ex { struct ib_uverbs_ioctl_hdr hdr; - struct ib_uverbs_attr attrs[QUERY_DEVICE_CAP_RESERVED]; + struct ib_uverbs_attr attrs[QUERY_DEVICE_RESERVED]; } __attribute__((packed, aligned(4))); int ibv_cmd_query_device_ex(struct ibv_context *context,