leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

RDMA/rxe: Separate HW and SW l/rkeys

Browse Source 
Separate software and simulated hardware lkeys and rkeys for MRs and MWs.
This makes struct ib_mr and struct ib_mw isolated from hardware changes
triggered by executing work requests.

This change fixes a bug seen in blktest.

Link: https://lore.kernel.org/r/20210914164206.19768-4-rpearsonhpe@gmail.com
Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
This commit is contained in:
Bob Pearson 2021-09-14 11:42:05 -05:00 committed by Jason Gunthorpe
parent 47b7f7064b
commit 001345339f
5 changed files with 81 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/infiniband/sw/rxe/rxe_loc.h
View File

@ -86,6 +86,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
void rxe_mr_cleanup(struct rxe_pool_entry *arg);

69
drivers/infiniband/sw/rxe/rxe_mr.c
View File

@ -53,8 +53,14 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1);
u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
mr->ibmr.lkey = lkey;
mr->ibmr.rkey = rkey;
/* set ibmr->l/rkey and also copy into private l/rkey
* for user MRs these will always be the same
* for cases where caller 'owns' the key portion
* they may be different until REG_MR WQE is executed.
*/
mr->lkey = mr->ibmr.lkey = lkey;
mr->rkey = mr->ibmr.rkey = rkey;
mr->state = RXE_MR_STATE_INVALID;
mr->map_shift = ilog2(RXE_BUF_PER_MAP);
}
@ -195,10 +201,8 @@ int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr)
{
int err;
rxe_mr_init(0, mr);
/* In fastreg, we also set the rkey */
mr->ibmr.rkey = mr->ibmr.lkey;
/* always allow remote access for FMRs */
rxe_mr_init(IB_ACCESS_REMOTE, mr);
err = rxe_mr_alloc(mr, max_pages);
if (err)
@ -511,8 +515,8 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
if (!mr)
return NULL;
if (unlikely((type == RXE_LOOKUP_LOCAL && mr_lkey(mr) != key) ||
(type == RXE_LOOKUP_REMOTE && mr_rkey(mr) != key) ||
if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||
(type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||
mr_pd(mr) != pd || (access && !(access & mr->access)) ||
mr->state != RXE_MR_STATE_VALID)) {
rxe_drop_ref(mr);
@ -535,9 +539,9 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
goto err;
}
if (rkey != mr->ibmr.rkey) {
pr_err("%s: rkey (%#x) doesn't match mr->ibmr.rkey (%#x)\n",
__func__, rkey, mr->ibmr.rkey);
if (rkey != mr->rkey) {
pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
__func__, rkey, mr->rkey);
ret = -EINVAL;
goto err_drop_ref;
}
@ -558,6 +562,49 @@ err:
return ret;
}
/* user can (re)register fast MR by executing a REG_MR WQE.
* user is expected to hold a reference on the ib mr until the
* WQE completes.
* Once a fast MR is created this is the only way to change the
* private keys. It is the responsibility of the user to maintain
* the ib mr keys in sync with rxe mr keys.
*/
int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
{
struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);
u32 key = wqe->wr.wr.reg.key;
u32 access = wqe->wr.wr.reg.access;
/* user can only register MR in free state */
if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
pr_warn("%s: mr->lkey = 0x%x not free\n",
__func__, mr->lkey);
return -EINVAL;
}
/* user can only register mr with qp in same protection domain */
if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
pr_warn("%s: qp->pd and mr->pd don't match\n",
__func__);
return -EINVAL;
}
/* user is only allowed to change key portion of l/rkey */
if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
pr_warn("%s: key = 0x%x has wrong index mr->lkey = 0x%x\n",
__func__, key, mr->lkey);
return -EINVAL;
}
mr->access = access;
mr->lkey = key;
mr->rkey = (access & IB_ACCESS_REMOTE) ? key : 0;
mr->iova = wqe->wr.wr.reg.mr->iova;
mr->state = RXE_MR_STATE_VALID;
return 0;
}
int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
{
struct rxe_mr *mr = to_rmr(ibmr);

30
drivers/infiniband/sw/rxe/rxe_mw.c
View File

@ -21,7 +21,7 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
}
rxe_add_index(mw);
ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
spin_lock_init(&mw->lock);
@ -71,6 +71,8 @@ int rxe_dealloc_mw(struct ib_mw *ibmw)
static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
struct rxe_mw *mw, struct rxe_mr *mr)
{
u32 key = wqe->wr.wr.mw.rkey & 0xff;
if (mw->ibmw.type == IB_MW_TYPE_1) {
if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
pr_err_once(
@ -108,7 +110,7 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
}
}
if (unlikely((wqe->wr.wr.mw.rkey & 0xff) == (mw->ibmw.rkey & 0xff))) {
if (unlikely(key == (mw->rkey & 0xff))) {
pr_err_once("attempt to bind MW with same key\n");
return -EINVAL;
}
@ -161,13 +163,9 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
struct rxe_mw *mw, struct rxe_mr *mr)
{
u32 rkey;
u32 new_rkey;
u32 key = wqe->wr.wr.mw.rkey & 0xff;
rkey = mw->ibmw.rkey;
new_rkey = (rkey & 0xffffff00) | (wqe->wr.wr.mw.rkey & 0x000000ff);
mw->ibmw.rkey = new_rkey;
mw->rkey = (mw->rkey & ~0xff) | key;
mw->access = wqe->wr.wr.mw.access;
mw->state = RXE_MW_STATE_VALID;
mw->addr = wqe->wr.wr.mw.addr;
@ -197,29 +195,29 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
struct rxe_mw *mw;
struct rxe_mr *mr;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;
u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
unsigned long flags;
mw = rxe_pool_get_index(&rxe->mw_pool,
wqe->wr.wr.mw.mw_rkey >> 8);
mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);
if (unlikely(!mw)) {
ret = -EINVAL;
goto err;
}
if (unlikely(mw->ibmw.rkey != wqe->wr.wr.mw.mw_rkey)) {
if (unlikely(mw->rkey != mw_rkey)) {
ret = -EINVAL;
goto err_drop_mw;
}
if (likely(wqe->wr.wr.mw.length)) {
mr = rxe_pool_get_index(&rxe->mr_pool,
wqe->wr.wr.mw.mr_lkey >> 8);
mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);
if (unlikely(!mr)) {
ret = -EINVAL;
goto err_drop_mw;
}
if (unlikely(mr->ibmr.lkey != wqe->wr.wr.mw.mr_lkey)) {
if (unlikely(mr->lkey != mr_lkey)) {
ret = -EINVAL;
goto err_drop_mr;
}
@ -292,7 +290,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
goto err;
}
if (rkey != mw->ibmw.rkey) {
if (rkey != mw->rkey) {
ret = -EINVAL;
goto err_drop_ref;
}
@ -323,7 +321,7 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)
if (!mw)
return NULL;
if (unlikely((rxe_mw_rkey(mw) != rkey) || rxe_mw_pd(mw) != pd ||
if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||
(mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||
(mw->length == 0) ||
(access && !(access & mw->access)) ||

14
drivers/infiniband/sw/rxe/rxe_req.c
View File

@ -562,7 +562,6 @@ static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
{
u8 opcode = wqe->wr.opcode;
struct rxe_mr *mr;
u32 rkey;
int ret;
@ -580,14 +579,11 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
}
break;
case IB_WR_REG_MR:
mr = to_rmr(wqe->wr.wr.reg.mr);
rxe_add_ref(mr);
mr->state = RXE_MR_STATE_VALID;
mr->access = wqe->wr.wr.reg.access;
mr->ibmr.lkey = wqe->wr.wr.reg.key;
mr->ibmr.rkey = wqe->wr.wr.reg.key;
mr->iova = wqe->wr.wr.reg.mr->iova;
rxe_drop_ref(mr);
ret = rxe_reg_fast_mr(qp, wqe);
if (unlikely(ret)) {
wqe->status = IB_WC_LOC_QP_OP_ERR;
return ret;
}
break;
case IB_WR_BIND_MW:
ret = rxe_bind_mw(qp, wqe);

18
drivers/infiniband/sw/rxe/rxe_verbs.h
View File

@ -305,6 +305,8 @@ struct rxe_mr {
struct ib_umem *umem;
u32 lkey;
u32 rkey;
enum rxe_mr_state state;
enum ib_mr_type type;
u64 va;
@ -342,6 +344,7 @@ struct rxe_mw {
enum rxe_mw_state state;
struct rxe_qp *qp; /* Type 2 only */
struct rxe_mr *mr;
u32 rkey;
int access;
u64 addr;
u64 length;
@ -466,26 +469,11 @@ static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
return to_rpd(mr->ibmr.pd);
}
static inline u32 mr_lkey(struct rxe_mr *mr)
{
return mr->ibmr.lkey;
}
static inline u32 mr_rkey(struct rxe_mr *mr)
{
return mr->ibmr.rkey;
}
static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
{
return to_rpd(mw->ibmw.pd);
}
static inline u32 rxe_mw_rkey(struct rxe_mw *mw)
{
return mw->ibmw.rkey;
}
int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
void rxe_mc_cleanup(struct rxe_pool_entry *arg);