leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

svcrdma: Change DMA mapping logic to avoid the page_address kernel API

Browse Source 
There was logic in the send path that assumed that a page containing data
to send to the client has a KVA. This is not always the case and can result
in data corruption when page_address returns zero and we end up DMA mapping
zero.

This patch changes the bus mapping logic to avoid page_address() where
necessary and converts all calls from ib_dma_map_single to ib_dma_map_page
in order to keep the map/unmap calls symmetric.

Signed-off-by: Tom Tucker <tom@ogc.us>
Signed-off-by: J. Bruce Fields <bfields@redhat.com>
This commit is contained in:
Tom Tucker 2010-10-12 15:33:52 -05:00 committed by J. Bruce Fields
parent ecec6e34e1
commit b432e6b3d9
3 changed files with 78 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
View File

@ -263,8 +263,8 @@ static int fast_reg_read_chunks(struct svcxprt_rdma *xprt,
frmr->page_list_len = PAGE_ALIGN(byte_count) >> PAGE_SHIFT; frmr->page_list_len = PAGE_ALIGN(byte_count) >> PAGE_SHIFT;
for (page_no = 0; page_no < frmr->page_list_len; page_no++) { for (page_no = 0; page_no < frmr->page_list_len; page_no++) {
frmr->page_list->page_list[page_no] = frmr->page_list->page_list[page_no] =
ib_dma_map_single(xprt->sc_cm_id->device, ib_dma_map_page(xprt->sc_cm_id->device,
page_address(rqstp->rq_arg.pages[page_no]), rqstp->rq_arg.pages[page_no], 0,
PAGE_SIZE, DMA_FROM_DEVICE); PAGE_SIZE, DMA_FROM_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, if (ib_dma_mapping_error(xprt->sc_cm_id->device,
frmr->page_list->page_list[page_no])) frmr->page_list->page_list[page_no]))
@ -309,15 +309,19 @@ static int rdma_set_ctxt_sge(struct svcxprt_rdma *xprt,
int count) int count)
{ {
int i; int i;
unsigned long off;
ctxt->count = count; ctxt->count = count;
ctxt->direction = DMA_FROM_DEVICE; ctxt->direction = DMA_FROM_DEVICE;
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
ctxt->sge[i].length = 0; /* in case map fails */ ctxt->sge[i].length = 0; /* in case map fails */
if (!frmr) { if (!frmr) {
BUG_ON(0 == virt_to_page(vec[i].iov_base));
off = (unsigned long)vec[i].iov_base & ~PAGE_MASK;
ctxt->sge[i].addr = ctxt->sge[i].addr =
ib_dma_map_single(xprt->sc_cm_id->device, ib_dma_map_page(xprt->sc_cm_id->device,
vec[i].iov_base, virt_to_page(vec[i].iov_base),
off,
vec[i].iov_len, vec[i].iov_len,
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, if (ib_dma_mapping_error(xprt->sc_cm_id->device,

68
net/sunrpc/xprtrdma/svc_rdma_sendto.c
View File

@ -96,21 +96,25 @@ static int fast_reg_xdr(struct svcxprt_rdma *xprt,
vec->count = 2; vec->count = 2;
sge_no++; sge_no++;
/* Build the FRMR */ /* Map the XDR head */
frmr->kva = frva; frmr->kva = frva;
frmr->direction = DMA_TO_DEVICE; frmr->direction = DMA_TO_DEVICE;
frmr->access_flags = 0; frmr->access_flags = 0;
frmr->map_len = PAGE_SIZE; frmr->map_len = PAGE_SIZE;
frmr->page_list_len = 1; frmr->page_list_len = 1;
page_off = (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK;
frmr->page_list->page_list[page_no] = frmr->page_list->page_list[page_no] =
ib_dma_map_single(xprt->sc_cm_id->device, ib_dma_map_page(xprt->sc_cm_id->device,
(void *)xdr->head[0].iov_base, virt_to_page(xdr->head[0].iov_base),
PAGE_SIZE, DMA_TO_DEVICE); page_off,
PAGE_SIZE - page_off,
DMA_TO_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, if (ib_dma_mapping_error(xprt->sc_cm_id->device,
frmr->page_list->page_list[page_no])) frmr->page_list->page_list[page_no]))
goto fatal_err; goto fatal_err;
atomic_inc(&xprt->sc_dma_used); atomic_inc(&xprt->sc_dma_used);
/* Map the XDR page list */
page_off = xdr->page_base; page_off = xdr->page_base;
page_bytes = xdr->page_len + page_off; page_bytes = xdr->page_len + page_off;
if (!page_bytes) if (!page_bytes)
@ -128,9 +132,9 @@ static int fast_reg_xdr(struct svcxprt_rdma *xprt,
page_bytes -= sge_bytes; page_bytes -= sge_bytes;
frmr->page_list->page_list[page_no] = frmr->page_list->page_list[page_no] =
ib_dma_map_single(xprt->sc_cm_id->device, ib_dma_map_page(xprt->sc_cm_id->device,
page_address(page), page, page_off,
PAGE_SIZE, DMA_TO_DEVICE); sge_bytes, DMA_TO_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, if (ib_dma_mapping_error(xprt->sc_cm_id->device,
frmr->page_list->page_list[page_no])) frmr->page_list->page_list[page_no]))
goto fatal_err; goto fatal_err;
@ -166,7 +170,9 @@ static int fast_reg_xdr(struct svcxprt_rdma *xprt,
vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off;
frmr->page_list->page_list[page_no] = frmr->page_list->page_list[page_no] =
ib_dma_map_single(xprt->sc_cm_id->device, va, PAGE_SIZE, ib_dma_map_page(xprt->sc_cm_id->device, virt_to_page(va),
page_off,
PAGE_SIZE,
DMA_TO_DEVICE); DMA_TO_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, if (ib_dma_mapping_error(xprt->sc_cm_id->device,
frmr->page_list->page_list[page_no])) frmr->page_list->page_list[page_no]))
@ -245,6 +251,35 @@ static int map_xdr(struct svcxprt_rdma *xprt,
return 0; return 0;
} }
static dma_addr_t dma_map_xdr(struct svcxprt_rdma *xprt,
struct xdr_buf *xdr,
u32 xdr_off, size_t len, int dir)
{
struct page *page;
dma_addr_t dma_addr;
if (xdr_off < xdr->head[0].iov_len) {
/* This offset is in the head */
xdr_off += (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK;
page = virt_to_page(xdr->head[0].iov_base);
} else {
xdr_off -= xdr->head[0].iov_len;
if (xdr_off < xdr->page_len) {
/* This offset is in the page list */
page = xdr->pages[xdr_off >> PAGE_SHIFT];
xdr_off &= ~PAGE_MASK;
} else {
/* This offset is in the tail */
xdr_off -= xdr->page_len;
xdr_off += (unsigned long)
xdr->tail[0].iov_base & ~PAGE_MASK;
page = virt_to_page(xdr->tail[0].iov_base);
}
}
dma_addr = ib_dma_map_page(xprt->sc_cm_id->device, page, xdr_off,
min_t(size_t, PAGE_SIZE, len), dir);
return dma_addr;
}
/* Assumptions: /* Assumptions:
* - We are using FRMR * - We are using FRMR
* - or - * - or -
@ -293,10 +328,9 @@ static int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp,
sge[sge_no].length = sge_bytes; sge[sge_no].length = sge_bytes;
if (!vec->frmr) { if (!vec->frmr) {
sge[sge_no].addr = sge[sge_no].addr =
ib_dma_map_single(xprt->sc_cm_id->device, dma_map_xdr(xprt, &rqstp->rq_res, xdr_off,
(void *)
vec->sge[xdr_sge_no].iov_base + sge_off,
sge_bytes, DMA_TO_DEVICE); sge_bytes, DMA_TO_DEVICE);
xdr_off += sge_bytes;
if (ib_dma_mapping_error(xprt->sc_cm_id->device, if (ib_dma_mapping_error(xprt->sc_cm_id->device,
sge[sge_no].addr)) sge[sge_no].addr))
goto err; goto err;
@ -494,7 +528,8 @@ static int send_reply_chunks(struct svcxprt_rdma *xprt,
* In all three cases, this function prepares the RPCRDMA header in * In all three cases, this function prepares the RPCRDMA header in
* sge[0], the 'type' parameter indicates the type to place in the * sge[0], the 'type' parameter indicates the type to place in the
* RPCRDMA header, and the 'byte_count' field indicates how much of * RPCRDMA header, and the 'byte_count' field indicates how much of
* the XDR to include in this RDMA_SEND. * the XDR to include in this RDMA_SEND. NB: The offset of the payload
* to send is zero in the XDR.
*/ */
static int send_reply(struct svcxprt_rdma *rdma, static int send_reply(struct svcxprt_rdma *rdma,
struct svc_rqst *rqstp, struct svc_rqst *rqstp,
@ -536,7 +571,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
ctxt->sge[0].lkey = rdma->sc_dma_lkey; ctxt->sge[0].lkey = rdma->sc_dma_lkey;
ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp);
ctxt->sge[0].addr = ctxt->sge[0].addr =
ib_dma_map_single(rdma->sc_cm_id->device, page_address(page), ib_dma_map_page(rdma->sc_cm_id->device, page, 0,
ctxt->sge[0].length, DMA_TO_DEVICE); ctxt->sge[0].length, DMA_TO_DEVICE);
if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr))
goto err; goto err;
@ -544,15 +579,16 @@ static int send_reply(struct svcxprt_rdma *rdma,
ctxt->direction = DMA_TO_DEVICE; ctxt->direction = DMA_TO_DEVICE;
/* Determine how many of our SGE are to be transmitted */ /* Map the payload indicated by 'byte_count' */
for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) {
int xdr_off = 0;
sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count);
byte_count -= sge_bytes; byte_count -= sge_bytes;
if (!vec->frmr) { if (!vec->frmr) {
ctxt->sge[sge_no].addr = ctxt->sge[sge_no].addr =
ib_dma_map_single(rdma->sc_cm_id->device, dma_map_xdr(rdma, &rqstp->rq_res, xdr_off,
vec->sge[sge_no].iov_base,
sge_bytes, DMA_TO_DEVICE); sge_bytes, DMA_TO_DEVICE);
xdr_off += sge_bytes;
if (ib_dma_mapping_error(rdma->sc_cm_id->device, if (ib_dma_mapping_error(rdma->sc_cm_id->device,
ctxt->sge[sge_no].addr)) ctxt->sge[sge_no].addr))
goto err; goto err;

16
net/sunrpc/xprtrdma/svc_rdma_transport.c
View File

@ -121,7 +121,7 @@ void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt)
*/ */
if (ctxt->sge[i].lkey == xprt->sc_dma_lkey) { if (ctxt->sge[i].lkey == xprt->sc_dma_lkey) {
atomic_dec(&xprt->sc_dma_used); atomic_dec(&xprt->sc_dma_used);
ib_dma_unmap_single(xprt->sc_cm_id->device, ib_dma_unmap_page(xprt->sc_cm_id->device,
ctxt->sge[i].addr, ctxt->sge[i].addr,
ctxt->sge[i].length, ctxt->sge[i].length,
ctxt->direction); ctxt->direction);
@ -503,8 +503,8 @@ int svc_rdma_post_recv(struct svcxprt_rdma *xprt)
BUG_ON(sge_no >= xprt->sc_max_sge); BUG_ON(sge_no >= xprt->sc_max_sge);
page = svc_rdma_get_page(); page = svc_rdma_get_page();
ctxt->pages[sge_no] = page; ctxt->pages[sge_no] = page;
pa = ib_dma_map_single(xprt->sc_cm_id->device, pa = ib_dma_map_page(xprt->sc_cm_id->device,
page_address(page), PAGE_SIZE, page, 0, PAGE_SIZE,
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, pa)) if (ib_dma_mapping_error(xprt->sc_cm_id->device, pa))
goto err_put_ctxt; goto err_put_ctxt;
@ -800,7 +800,7 @@ static void frmr_unmap_dma(struct svcxprt_rdma *xprt,
if (ib_dma_mapping_error(frmr->mr->device, addr)) if (ib_dma_mapping_error(frmr->mr->device, addr))
continue; continue;
atomic_dec(&xprt->sc_dma_used); atomic_dec(&xprt->sc_dma_used);
ib_dma_unmap_single(frmr->mr->device, addr, PAGE_SIZE, ib_dma_unmap_page(frmr->mr->device, addr, PAGE_SIZE,
frmr->direction); frmr->direction);
} }
} }
@ -1276,7 +1276,7 @@ int svc_rdma_send(struct svcxprt_rdma *xprt, struct ib_send_wr *wr)
atomic_read(&xprt->sc_sq_count) < atomic_read(&xprt->sc_sq_count) <
xprt->sc_sq_depth); xprt->sc_sq_depth);
if (test_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags)) if (test_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags))
return 0; return -ENOTCONN;
continue; continue;
} }
/* Take a transport ref for each WR posted */ /* Take a transport ref for each WR posted */
@ -1322,8 +1322,8 @@ void svc_rdma_send_error(struct svcxprt_rdma *xprt, struct rpcrdma_msg *rmsgp,
length = svc_rdma_xdr_encode_error(xprt, rmsgp, err, va); length = svc_rdma_xdr_encode_error(xprt, rmsgp, err, va);
/* Prepare SGE for local address */ /* Prepare SGE for local address */
sge.addr = ib_dma_map_single(xprt->sc_cm_id->device, sge.addr = ib_dma_map_page(xprt->sc_cm_id->device,
page_address(p), PAGE_SIZE, DMA_FROM_DEVICE); p, 0, PAGE_SIZE, DMA_FROM_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, sge.addr)) { if (ib_dma_mapping_error(xprt->sc_cm_id->device, sge.addr)) {
put_page(p); put_page(p);
return; return;
@ -1350,7 +1350,7 @@ void svc_rdma_send_error(struct svcxprt_rdma *xprt, struct rpcrdma_msg *rmsgp,
if (ret) { if (ret) {
dprintk("svcrdma: Error %d posting send for protocol error\n", dprintk("svcrdma: Error %d posting send for protocol error\n",
ret); ret);
ib_dma_unmap_single(xprt->sc_cm_id->device, ib_dma_unmap_page(xprt->sc_cm_id->device,
sge.addr, PAGE_SIZE, sge.addr, PAGE_SIZE,
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
svc_rdma_put_context(ctxt, 1); svc_rdma_put_context(ctxt, 1);