Skip to content

Abi poc1 #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yonicswe wants to merge 4 commits into
base: abi_poc1
Choose a base branch
from
Open

Abi poc1 #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
84cf060
fix compilation ptr_idr
yonatanco Mar 12, 2017
5deedc5
Add query_qp verb
yonatanco Mar 8, 2017
d6e9d0a
Add rereg_mr verb
yonatanco Mar 12, 2017
b177922
Merge user kernel headers
yonatanco Mar 15, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 1 addition & 166 deletions include/infiniband/kern-abi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#define KERN_ABI_H

#include <linux/types.h>
#include <rdma/rdma_user_ioctl_cmd.h>

/*
* This file must be kept in sync with the kernel's version of
Expand Down Expand Up @@ -1168,172 +1169,6 @@ enum {

#define UVERBS_UHW_SZ ((UVERBS_UHW_OUT & ~IB_UVERBS_SCOPE_DRIVER) + 1)

enum uverbs_common_types {
UVERBS_TYPE_DEVICE, /* Don't use IDRs here */
UVERBS_TYPE_PD,
UVERBS_TYPE_COMP_CHANNEL,
UVERBS_TYPE_CQ,
UVERBS_TYPE_QP,
UVERBS_TYPE_SRQ,
UVERBS_TYPE_AH,
UVERBS_TYPE_MR,
UVERBS_TYPE_MW,
UVERBS_TYPE_FLOW,
UVERBS_TYPE_XRCD,
UVERBS_TYPE_LAST,
};

enum uverbs_create_qp_cmd_attr {
CREATE_QP_HANDLE,
CREATE_QP_PD_HANDLE,
CREATE_QP_SEND_CQ,
CREATE_QP_RECV_CQ,
CREATE_QP_SRQ,
CREATE_QP_USER_HANDLE,
CREATE_QP_CMD,
CREATE_QP_CMD_FLAGS,
CREATE_QP_RESP,
CREATE_QP_RESERVED
};

enum uverbs_destroy_qp_cmd_attr {
DESTROY_QP_HANDLE,
DESTROY_QP_EVENTS_REPORTED,
DESTROY_QP_RESERVED
};

enum uverbs_create_cq_cmd_attr {
CREATE_CQ_HANDLE,
CREATE_CQ_CQE,
CREATE_CQ_USER_HANDLE,
CREATE_CQ_COMP_CHANNEL,
CREATE_CQ_COMP_VECTOR,
CREATE_CQ_FLAGS,
CREATE_CQ_RESP_CQE,
CREATE_CQ_RESERVED,
};

enum uverbs_destroy_cq_cmd_attr {
DESTROY_CQ_HANDLE,
DESTROY_CQ_RESP,
DESTROY_CQ_RESERVED
};

enum uverbs_create_qp_xrc_tgt_cmd_attr {
CREATE_QP_XRC_TGT_HANDLE,
CREATE_QP_XRC_TGT_XRCD,
CREATE_QP_XRC_TGT_USER_HANDLE,
CREATE_QP_XRC_TGT_CMD,
CREATE_QP_XRC_TGT_CMD_FLAGS,
CREATE_QP_XRC_TGT_RESP,
CREATE_QP_XRC_TGT_RESERVED
};

enum uverbs_modify_qp_cmd_attr {
MODIFY_QP_HANDLE,
MODIFY_QP_STATE,
MODIFY_QP_CUR_STATE,
MODIFY_QP_EN_SQD_ASYNC_NOTIFY,
MODIFY_QP_ACCESS_FLAGS,
MODIFY_QP_PKEY_INDEX,
MODIFY_QP_PORT,
MODIFY_QP_QKEY,
MODIFY_QP_AV,
MODIFY_QP_PATH_MTU,
MODIFY_QP_TIMEOUT,
MODIFY_QP_RETRY_CNT,
MODIFY_QP_RNR_RETRY,
MODIFY_QP_RQ_PSN,
MODIFY_QP_MAX_RD_ATOMIC,
MODIFY_QP_ALT_PATH,
MODIFY_QP_MIN_RNR_TIMER,
MODIFY_QP_SQ_PSN,
MODIFY_QP_MAX_DEST_RD_ATOMIC,
MODIFY_QP_PATH_MIG_STATE,
MODIFY_QP_DEST_QPN,
MODIFY_QP_RESERVED
};

enum uverbs_create_comp_channel_cmd_attr {
CREATE_COMP_CHANNEL_FD,
CREATE_COMP_CHANNEL_RESERVED
};

enum uverbs_get_context {
GET_CONTEXT_RESP,
GET_CONTEXT_RESERVED,
};

enum uverbs_query_device {
QUERY_DEVICE_RESP,
QUERY_DEVICE_ODP,
QUERY_DEVICE_TIMESTAMP_MASK,
QUERY_DEVICE_HCA_CORE_CLOCK,
QUERY_DEVICE_CAP_FLAGS,
QUERY_DEVICE_CAP_RESERVED,
};

enum uverbs_query_port {
QUERY_PORT_PORT_NUM,
QUERY_PORT_RESP,
QUERY_PORT_RESERVED
};

enum uverbs_alloc_pd {
ALLOC_PD_HANDLE,
ALLOC_PD_RESERVED,
};

enum uverbs_dealloc_pd {
DEALLOC_PD_HANDLE,
DEALLOC_PD_RESERVED,
};

enum uverbs_reg_mr {
REG_MR_HANDLE,
REG_MR_PD_HANDLE,
REG_MR_CMD,
REG_MR_RESP,
REG_MR_RESERVED,
};

enum uverbs_dereg_mr {
DEREG_MR_HANDLE,
DEREG_MR_RESERVED
};

enum uverbs_actions_mr_ops {
UVERBS_MR_REG,
UVERBS_MR_DEREG,
};

enum uverbs_actions_comp_channel_ops {
UVERBS_COMP_CHANNEL_CREATE,
};

enum uverbs_actions_cq_ops {
UVERBS_CQ_CREATE,
UVERBS_CQ_DESTROY,
};

enum uverbs_actions_qp_ops {
UVERBS_QP_CREATE,
UVERBS_QP_CREATE_XRC_TGT,
UVERBS_QP_MODIFY,
UVERBS_QP_DESTROY
};

enum uverbs_actions_pd_ops {
UVERBS_PD_ALLOC,
UVERBS_PD_DEALLOC,
};

enum uverbs_actions_device_ops {
UVERBS_DEVICE_ALLOC_CONTEXT,
UVERBS_DEVICE_QUERY,
UVERBS_DEVICE_PORT_QUERY,
};

struct ib_uverbs_ioctl_create_qp {
__u32 max_send_wr;
__u32 max_recv_wr;
Expand Down
99 changes: 85 additions & 14 deletions src/cmd.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *leg
attr - cmd.attrs);

ret = ioctl(context->cmd_fd, RDMA_VERBS_IOCTL, &cmd);
printf("ret is %d\n", ret);
printf("ret is %lu\n", ret);
if (ret)
return errno;

Expand Down Expand Up @@ -163,7 +163,7 @@ int ibv_cmd_query_device(struct ibv_context *context,

struct ibv_ioctl_cmd_query_device_ex {
struct ib_uverbs_ioctl_hdr hdr;
struct ib_uverbs_attr attrs[QUERY_DEVICE_CAP_RESERVED];
struct ib_uverbs_attr attrs[QUERY_DEVICE_RESERVED];
} __attribute__((packed, aligned(4)));

int ibv_cmd_query_device_ex(struct ibv_context *context,
Expand Down Expand Up @@ -313,7 +313,7 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,

(void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);

pd->handle = cmd.attrs[0].ptr_idr;
pd->handle = cmd.attrs[0].data;
pd->context = context;

return 0;
Expand Down Expand Up @@ -433,14 +433,72 @@ int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,

(void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);

mr->handle = cmd.attrs[0].ptr_idr;
mr->handle = cmd.attrs[0].data;
mr->lkey = resp.lkey;
mr->rkey = resp.rkey;
mr->context = pd->context;

return 0;
}

#define new_rereg_me
#ifdef new_rereg_me
struct ibv_ioctl_rereg_mr_cmd {
__u64 response;
__u32 mr_handle;
__u32 flags;
__u64 start;
__u64 length;
__u64 hca_va;
__u32 pd_handle;
__u32 access_flags;
};

struct ibv_ioctl_cmd_rereg_mr {
struct ib_uverbs_ioctl_hdr hdr;
struct ib_uverbs_attr attrs[REREG_MR_RESERVED];
}__attribute__((packed, aligned(4)));

int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr,
size_t length, uint64_t hca_va, int access,
struct ibv_pd *pd, struct ibv_rereg_mr *cmd,
size_t cmd_sz, struct ibv_rereg_mr_resp *resp,
size_t resp_sz)
{
struct ibv_ioctl_cmd_rereg_mr iocmd;
struct ib_uverbs_attr *cattr = iocmd.attrs;
struct ibv_ioctl_rereg_mr_cmd rereg_mr_cmd;

fill_attr_obj(cattr++, REREG_MR_HANDLE, mr->handle);

if (flags & IBV_REREG_MR_CHANGE_PD)
fill_attr_obj(cattr++, REREG_MR_PD_HANDLE, pd->handle);
else
fill_attr_obj(cattr++, REREG_MR_PD_HANDLE, 0);

fill_attr_in(cattr++, REREG_MR_CMD, sizeof(struct ibv_ioctl_rereg_mr_cmd), &rereg_mr_cmd);
fill_attr_out(cattr++, REREG_MR_RESP, sizeof(struct ibv_rereg_mr_resp), resp);

rereg_mr_cmd.start = (uintptr_t)addr;
rereg_mr_cmd.length = length;
rereg_mr_cmd.hca_va = hca_va;
rereg_mr_cmd.flags = flags;
rereg_mr_cmd.access_flags = access;

fill_ioctl_hdr(&iocmd.hdr, UVERBS_TYPE_MR, (void *)cattr - (void *)&iocmd,
UVERBS_MR_REREG, cattr - iocmd.attrs);

if (ioctl(mr->context->cmd_fd, RDMA_VERBS_IOCTL, &iocmd))
return errno;

mr->lkey = resp->lkey;
mr->rkey = resp->rkey;
if (flags & IBV_REREG_MR_CHANGE_PD)
mr->context = pd->context;

return 0;
}
#else
int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr,
size_t length, uint64_t hca_va, int access,
struct ibv_pd *pd, struct ibv_rereg_mr *cmd,
Expand Down Expand Up @@ -469,6 +527,7 @@ int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr,

return 0;
}
#endif

struct ibv_ioctl_cmd_dereg_mr {
struct ib_uverbs_ioctl_hdr hdr;
Expand Down Expand Up @@ -577,7 +636,7 @@ int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
printf("%s:%d\n", __func__, __LINE__);
cq->context = context;
cq->cqe = _cqe_out;
cq->handle = cmd.attrs[0].ptr_idr;
cq->handle = cmd.attrs[0].data;
printf("%s:%d\n", __func__, __LINE__);

return 0;
Expand Down Expand Up @@ -1102,7 +1161,7 @@ int ibv_cmd_create_qp_ex2(struct ibv_context *context,

create_qp_ioctl_handle_resp_common(context, qp, qp_attr,
&resp,
cmd.attrs[0].ptr_idr,
cmd.attrs[0].data,
vxrcd,
vqp_sz);

Expand Down Expand Up @@ -1152,7 +1211,7 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context,

create_qp_ioctl_handle_resp_common(context, qp, attr_ex,
&resp,
cmd.attrs[0].ptr_idr,
cmd.attrs[0].data,
vxrcd,
vqp_sz);

Expand Down Expand Up @@ -1267,21 +1326,32 @@ int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp,
return 0;
}

int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
struct ibv_ioctl_cmd_query_qp {
struct ib_uverbs_ioctl_hdr hdr;
struct ib_uverbs_attr attrs[QUERY_QP_RESERVED];
} __attribute__((packed, aligned(4)));

int ibv_cmd_query_qp(struct ibv_qp *qp,
struct ibv_qp_attr *attr,
int attr_mask,
struct ibv_qp_init_attr *init_attr,
struct ibv_query_qp *cmd, size_t cmd_size)
{
struct ibv_ioctl_cmd_query_qp iocmd;
struct ib_uverbs_attr *cattr = iocmd.attrs;
struct ibv_query_qp_resp resp;
int ret;

IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp);
cmd->qp_handle = qp->handle;
cmd->attr_mask = attr_mask;
fill_attr_obj(cattr++, QUERY_QP_HANDLE, qp->handle);
fill_attr_in(cattr++, QUERY_QP_ATTR_MASK, sizeof(__u32), &attr_mask);
fill_attr_out(cattr++, QUERY_QP_RESP, sizeof(resp), &resp);

if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
return errno;
fill_ioctl_hdr(&iocmd.hdr, UVERBS_TYPE_QP, (void *)cattr - (void *)&iocmd,
UVERBS_QP_QUERY, cattr - iocmd.attrs);

(void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
ret = ioctl(qp->context->cmd_fd, RDMA_VERBS_IOCTL, &iocmd);
if (ret)
return errno;

attr->qkey = resp.qkey;
attr->rq_psn = resp.rq_psn;
Expand Down Expand Up @@ -1347,6 +1417,7 @@ int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
init_attr->sq_sig_all = resp.sq_sig_all;

return 0;

}

struct ibv_ioctl_cmd_modify_qp {
Expand Down
Loading