leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IB/rdmavt: Use per-CPU reference count for MRs

Browse Source 
Having per-CPU reference count for each MR prevents
cache-line bouncing across the system. Thus, it
prevents bottlenecks. Use per-CPU reference counts
per MR.

The per-CPU reference count for FMRs is used in
atomic mode to allow accurate testing of the busy
state. Other MR types run in per-CPU mode MR until
they're freed.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Sebastian Sanchez <sebastian.sanchez@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
This commit is contained in:
Sebastian Sanchez 2017-02-08 05:26:31 -08:00 committed by Doug Ledford
parent f3e862cb68
commit 338adfdddf
2 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
drivers/infiniband/sw/rdmavt/mr.c
View File

@ -120,10 +120,19 @@ static void rvt_deinit_mregion(struct rvt_mregion *mr)
mr->mapsz = 0; mr->mapsz = 0;
while (i) while (i)
kfree(mr->map[--i]); kfree(mr->map[--i]);
percpu_ref_exit(&mr->refcount);
}
static void __rvt_mregion_complete(struct percpu_ref *ref)
{
struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
refcount);
complete(&mr->comp);
} }
static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd, static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
int count) int count, unsigned int percpu_flags)
{ {
int m, i = 0; int m, i = 0;
struct rvt_dev_info *dev = ib_to_rvt(pd->device); struct rvt_dev_info *dev = ib_to_rvt(pd->device);
@ -133,19 +142,23 @@ static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
for (; i < m; i++) { for (; i < m; i++) {
mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL, mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
dev->dparms.node); dev->dparms.node);
if (!mr->map[i]) { if (!mr->map[i])
rvt_deinit_mregion(mr); goto bail;
return -ENOMEM;
}
mr->mapsz++; mr->mapsz++;
} }
init_completion(&mr->comp); init_completion(&mr->comp);
/* count returning the ptr to user */ /* count returning the ptr to user */
atomic_set(&mr->refcount, 1); if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
percpu_flags, GFP_KERNEL))
goto bail;
atomic_set(&mr->lkey_invalid, 0); atomic_set(&mr->lkey_invalid, 0);
mr->pd = pd; mr->pd = pd;
mr->max_segs = count; mr->max_segs = count;
return 0; return 0;
bail:
rvt_deinit_mregion(mr);
return -ENOMEM;
} }
/** /**
@ -180,8 +193,7 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
if (!tmr) { if (!tmr) {
rcu_assign_pointer(dev->dma_mr, mr); rcu_assign_pointer(dev->dma_mr, mr);
mr->lkey_published = 1; mr->lkey_published = 1;
} else { rvt_get_mr(mr);
rvt_put_mr(mr);
} }
goto success; goto success;
} }
@ -239,11 +251,14 @@ static void rvt_free_lkey(struct rvt_mregion *mr)
int freed = 0; int freed = 0;
spin_lock_irqsave(&rkt->lock, flags); spin_lock_irqsave(&rkt->lock, flags);
if (!mr->lkey_published) if (!lkey) {
goto out; if (mr->lkey_published) {
if (lkey == 0) { RCU_INIT_POINTER(dev->dma_mr, NULL);
RCU_INIT_POINTER(dev->dma_mr, NULL); rvt_put_mr(mr);
}
} else { } else {
if (!mr->lkey_published)
goto out;
r = lkey >> (32 - dev->dparms.lkey_table_size); r = lkey >> (32 - dev->dparms.lkey_table_size);
RCU_INIT_POINTER(rkt->table[r], NULL); RCU_INIT_POINTER(rkt->table[r], NULL);
} }
@ -253,7 +268,7 @@ out:
spin_unlock_irqrestore(&rkt->lock, flags); spin_unlock_irqrestore(&rkt->lock, flags);
if (freed) { if (freed) {
synchronize_rcu(); synchronize_rcu();
rvt_put_mr(mr); percpu_ref_kill(&mr->refcount);
} }
} }
@ -269,7 +284,7 @@ static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
if (!mr) if (!mr)
goto bail; goto bail;
rval = rvt_init_mregion(&mr->mr, pd, count); rval = rvt_init_mregion(&mr->mr, pd, count, 0);
if (rval) if (rval)
goto bail; goto bail;
/* /*
@ -294,8 +309,8 @@ bail:
static void __rvt_free_mr(struct rvt_mr *mr) static void __rvt_free_mr(struct rvt_mr *mr)
{ {
rvt_deinit_mregion(&mr->mr);
rvt_free_lkey(&mr->mr); rvt_free_lkey(&mr->mr);
rvt_deinit_mregion(&mr->mr);
kfree(mr); kfree(mr);
} }
@ -323,7 +338,7 @@ struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
goto bail; goto bail;
} }
rval = rvt_init_mregion(&mr->mr, pd, 0); rval = rvt_init_mregion(&mr->mr, pd, 0, 0);
if (rval) { if (rval) {
ret = ERR_PTR(rval); ret = ERR_PTR(rval);
goto bail; goto bail;
@ -445,8 +460,8 @@ int rvt_dereg_mr(struct ib_mr *ibmr)
timeout = wait_for_completion_timeout(&mr->mr.comp, 5 * HZ); timeout = wait_for_completion_timeout(&mr->mr.comp, 5 * HZ);
if (!timeout) { if (!timeout) {
rvt_pr_err(rdi, rvt_pr_err(rdi,
"rvt_dereg_mr timeout mr %p pd %p refcount %u\n", "rvt_dereg_mr timeout mr %p pd %p\n",
mr, mr->mr.pd, atomic_read(&mr->mr.refcount)); mr, mr->mr.pd);
rvt_get_mr(&mr->mr); rvt_get_mr(&mr->mr);
ret = -EBUSY; ret = -EBUSY;
goto out; goto out;
@ -623,7 +638,8 @@ struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
if (!fmr) if (!fmr)
goto bail; goto bail;
rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages); rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages,
PERCPU_REF_INIT_ATOMIC);
if (rval) if (rval)
goto bail; goto bail;
@ -674,11 +690,12 @@ int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
struct rvt_fmr *fmr = to_ifmr(ibfmr); struct rvt_fmr *fmr = to_ifmr(ibfmr);
struct rvt_lkey_table *rkt; struct rvt_lkey_table *rkt;
unsigned long flags; unsigned long flags;
int m, n, i; int m, n;
unsigned long i;
u32 ps; u32 ps;
struct rvt_dev_info *rdi = ib_to_rvt(ibfmr->device); struct rvt_dev_info *rdi = ib_to_rvt(ibfmr->device);
i = atomic_read(&fmr->mr.refcount); i = atomic_long_read(&fmr->mr.refcount.count);
if (i > 2) if (i > 2)
return -EBUSY; return -EBUSY;

10
include/rdma/rdmavt_mr.h
View File

@ -52,6 +52,7 @@
* For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once * For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
* drivers no longer need access to the MR directly. * drivers no longer need access to the MR directly.
*/ */
#include <linux/percpu-refcount.h>
/* /*
* A segment is a linear region of low physical memory. * A segment is a linear region of low physical memory.
@ -79,11 +80,11 @@ struct rvt_mregion {
int access_flags; int access_flags;
u32 max_segs; /* number of rvt_segs in all the arrays */ u32 max_segs; /* number of rvt_segs in all the arrays */
u32 mapsz; /* size of the map array */ u32 mapsz; /* size of the map array */
atomic_t lkey_invalid; /* true if current lkey is invalid */
u8 page_shift; /* 0 - non unform/non powerof2 sizes */ u8 page_shift; /* 0 - non unform/non powerof2 sizes */
u8 lkey_published; /* in global table */ u8 lkey_published; /* in global table */
atomic_t lkey_invalid; /* true if current lkey is invalid */ struct percpu_ref refcount;
struct completion comp; /* complete when refcount goes to zero */ struct completion comp; /* complete when refcount goes to zero */
atomic_t refcount;
struct rvt_segarray *map[0]; /* the segments */ struct rvt_segarray *map[0]; /* the segments */
}; };
@ -123,13 +124,12 @@ struct rvt_sge_state {
static inline void rvt_put_mr(struct rvt_mregion *mr) static inline void rvt_put_mr(struct rvt_mregion *mr)
{ {
if (unlikely(atomic_dec_and_test(&mr->refcount))) percpu_ref_put(&mr->refcount);
complete(&mr->comp);
} }
static inline void rvt_get_mr(struct rvt_mregion *mr) static inline void rvt_get_mr(struct rvt_mregion *mr)
{ {
atomic_inc(&mr->refcount); percpu_ref_get(&mr->refcount);
} }
static inline void rvt_put_ss(struct rvt_sge_state *ss) static inline void rvt_put_ss(struct rvt_sge_state *ss)