Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-21 19:41:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

cifs: Change the I/O paths to use an iterator rather than a page list

Browse Source 
Currently, the cifs I/O paths hand lists of pages from the VM interface
routines at the top all the way through the intervening layers to the
socket interface at the bottom.

This is a problem, however, for interfacing with netfslib which passes an
iterator through to the ->issue_read() method (and will pass an iterator
through to the ->issue_write() method in future).  Netfslib takes over
bounce buffering for direct I/O, async I/O and encrypted content, so cifs
doesn't need to do that.  Netfslib also converts IOVEC-type iterators into
BVEC-type iterators if necessary.

Further, cifs needs foliating - and folios may come in a variety of sizes,
so a page list pointing to an array of heterogeneous pages may cause
problems in places such as where crypto is done.

Change the cifs I/O paths to hand iov_iter iterators all the way through
instead.

Notes:

 (1) Some old routines are #if'd out to be removed in a follow up patch so
     as to avoid confusing diff, thereby making the diff output easier to
     follow.  I've removed functions that don't overlap with anything
     added.

 (2) struct smb_rqst loses rq_pages, rq_offset, rq_npages, rq_pagesz and
     rq_tailsz which describe the pages forming the buffer; instead there's
     an rq_iter describing the source buffer and an rq_buffer which is used
     to hold the buffer for encryption.

 (3) struct cifs_readdata and cifs_writedata are similarly modified to
     smb_rqst.  The ->read_into_pages() and ->copy_into_pages() are then
     replaced with passing the iterator directly to the socket.

     The iterators are stored in these structs so that they are persistent
     and don't get deallocated when the function returns (unlike if they
     were stack variables).

 (4) Buffered writeback is overhauled, borrowing the code from the afs
     filesystem to gather up contiguous runs of folios.  The XARRAY-type
     iterator is then used to refer directly to the pagecache and can be
     passed to the socket to transmit data directly from there.

     This includes:

	cifs_extend_writeback()
	cifs_write_back_from_locked_folio()
	cifs_writepages_region()
	cifs_writepages()

 (5) Pages are converted to folios.

 (6) Direct I/O uses netfs_extract_user_iter() to create a BVEC-type
     iterator from an IOBUF/UBUF-type source iterator.

 (7) smb2_get_aead_req() uses netfs_extract_iter_to_sg() to extract page
     fragments from the iterator into the scatterlists that the crypto
     layer prefers.

 (8) smb2_init_transform_rq() attached pages to smb_rqst::rq_buffer, an
     xarray, to use as a bounce buffer for encryption.  An XARRAY-type
     iterator can then be used to pass the bounce buffer to lower layers.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: Steve French <sfrench@samba.org>
cc: Shyam Prasad N <nspmangalore@gmail.com>
cc: Rohith Surabattula <rohiths.msft@gmail.com>
cc: Paulo Alcantara <pc@cjr.nz>
cc: Jeff Layton <jlayton@kernel.org>
cc: linux-cifs@vger.kernel.org

Link: https://lore.kernel.org/r/164311907995.2806745.400147335497304099.stgit@warthog.procyon.org.uk/ # rfc
Link: https://lore.kernel.org/r/164928620163.457102.11602306234438271112.stgit@warthog.procyon.org.uk/ # v1
Link: https://lore.kernel.org/r/165211420279.3154751.15923591172438186144.stgit@warthog.procyon.org.uk/ # v1
Link: https://lore.kernel.org/r/165348880385.2106726.3220789453472800240.stgit@warthog.procyon.org.uk/ # v1
Link: https://lore.kernel.org/r/165364827111.3334034.934805882842932881.stgit@warthog.procyon.org.uk/ # v3
Link: https://lore.kernel.org/r/166126396180.708021.271013668175370826.stgit@warthog.procyon.org.uk/ # v1
Link: https://lore.kernel.org/r/166697259595.61150.5982032408321852414.stgit@warthog.procyon.org.uk/ # rfc
Link: https://lore.kernel.org/r/166732031756.3186319.12528413619888902872.stgit@warthog.procyon.org.uk/ # rfc
Signed-off-by: Steve French <stfrench@microsoft.com>
This commit is contained in:
David Howells 2022-01-24 21:13:24 +00:00 committed by Steve French
parent 16541195c6
commit d08089f649
14 changed files with 1134 additions and 1092 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fs/cifs/Kconfig
View File

@ -18,6 +18,7 @@ config CIFS
select DNS_RESOLVER
select ASN1
select OID_REGISTRY
select NETFS_SUPPORT
help
This is the client VFS module for the SMB3 family of network file
protocols (including the most recent, most secure dialect SMB3.1.1).

28
fs/cifs/cifsencrypt.c
View File

@ -169,11 +169,11 @@ static int cifs_shash_iter(const struct iov_iter *iter, size_t maxsize,
}
int __cifs_calc_signature(struct smb_rqst *rqst,
struct TCP_Server_Info *server, char *signature,
struct shash_desc *shash)
struct TCP_Server_Info *server, char *signature,
struct shash_desc *shash)
{
int i;
int rc;
ssize_t rc;
struct kvec *iov = rqst->rq_iov;
int n_vec = rqst->rq_nvec;
@ -205,25 +205,9 @@ int __cifs_calc_signature(struct smb_rqst *rqst,
}
}
/* now hash over the rq_pages array */
for (i = 0; i < rqst->rq_npages; i++) {
void *kaddr;
unsigned int len, offset;
rqst_page_get_length(rqst, i, &len, &offset);
kaddr = (char *) kmap(rqst->rq_pages[i]) + offset;
rc = crypto_shash_update(shash, kaddr, len);
if (rc) {
cifs_dbg(VFS, "%s: Could not update with payload\n",
__func__);
kunmap(rqst->rq_pages[i]);
return rc;
}
kunmap(rqst->rq_pages[i]);
}
rc = cifs_shash_iter(&rqst->rq_iter, iov_iter_count(&rqst->rq_iter), shash);
if (rc < 0)
return rc;
rc = crypto_shash_final(shash, signature);
if (rc)

66
fs/cifs/cifsglob.h
View File

@ -212,11 +212,9 @@ static inline void cifs_free_open_info(struct cifs_open_info_data *data)
struct smb_rqst {
struct kvec *rq_iov; /* array of kvecs */
unsigned int rq_nvec; /* number of kvecs in array */
struct page **rq_pages; /* pointer to array of page ptrs */
unsigned int rq_offset; /* the offset to the 1st page */
unsigned int rq_npages; /* number pages in array */
unsigned int rq_pagesz; /* page size to use */
unsigned int rq_tailsz; /* length of last page */
size_t rq_iter_size; /* Amount of data in ->rq_iter */
struct iov_iter rq_iter; /* Data iterator */
struct xarray rq_buffer; /* Page buffer for encryption */
};
struct mid_q_entry;
@ -1421,10 +1419,11 @@ struct cifs_aio_ctx {
struct cifsFileInfo *cfile;
struct bio_vec *bv;
loff_t pos;
unsigned int npages;
unsigned int nr_pinned_pages;
ssize_t rc;
unsigned int len;
unsigned int total_len;
unsigned int bv_need_unpin; /* If ->bv[] needs unpinning */
bool should_dirty;
/*
* Indicates if this aio_ctx is for direct_io,
@ -1442,28 +1441,18 @@ struct cifs_readdata {
struct address_space *mapping;
struct cifs_aio_ctx *ctx;
__u64 offset;
ssize_t got_bytes;
unsigned int bytes;
unsigned int got_bytes;
pid_t pid;
int result;
struct work_struct work;
int (*read_into_pages)(struct TCP_Server_Info *server,
struct cifs_readdata *rdata,
unsigned int len);
int (*copy_into_pages)(struct TCP_Server_Info *server,
struct cifs_readdata *rdata,
struct iov_iter *iter);
struct iov_iter iter;
struct kvec iov[2];
struct TCP_Server_Info *server;
#ifdef CONFIG_CIFS_SMB_DIRECT
struct smbd_mr *mr;
#endif
unsigned int pagesz;
unsigned int page_offset;
unsigned int tailsz;
struct cifs_credits credits;
unsigned int nr_pages;
struct page **pages;
};
/* asynchronous write support */
@ -1475,6 +1464,8 @@ struct cifs_writedata {
struct work_struct work;
struct cifsFileInfo *cfile;
struct cifs_aio_ctx *ctx;
struct iov_iter iter;
struct bio_vec *bv;
__u64 offset;
pid_t pid;
unsigned int bytes;
@ -1483,12 +1474,7 @@ struct cifs_writedata {
#ifdef CONFIG_CIFS_SMB_DIRECT
struct smbd_mr *mr;
#endif
unsigned int pagesz;
unsigned int page_offset;
unsigned int tailsz;
struct cifs_credits credits;
unsigned int nr_pages;
struct page **pages;
};
/*
@ -2148,9 +2134,9 @@ static inline void move_cifs_info_to_smb2(struct smb2_file_all_info *dst, const
dst->FileNameLength = src->FileNameLength;
}
static inline unsigned int cifs_get_num_sgs(const struct smb_rqst *rqst,
int num_rqst,
const u8 *sig)
static inline int cifs_get_num_sgs(const struct smb_rqst *rqst,
int num_rqst,
const u8 *sig)
{
unsigned int len, skip;
unsigned int nents = 0;
@ -2170,6 +2156,19 @@ static inline unsigned int cifs_get_num_sgs(const struct smb_rqst *rqst,
* rqst[1+].rq_iov[0+] data to be encrypted/decrypted
*/
for (i = 0; i < num_rqst; i++) {
/* We really don't want a mixture of pinned and unpinned pages
* in the sglist. It's hard to keep track of which is what.
* Instead, we convert to a BVEC-type iterator higher up.
*/
if (WARN_ON_ONCE(user_backed_iter(&rqst[i].rq_iter)))
return -EIO;
/* We also don't want to have any extra refs or pins to clean
* up in the sglist.
*/
if (WARN_ON_ONCE(iov_iter_extract_will_pin(&rqst[i].rq_iter)))
return -EIO;
for (j = 0; j < rqst[i].rq_nvec; j++) {
struct kvec *iov = &rqst[i].rq_iov[j];
@ -2183,7 +2182,7 @@ static inline unsigned int cifs_get_num_sgs(const struct smb_rqst *rqst,
}
skip = 0;
}
nents += rqst[i].rq_npages;
nents += iov_iter_npages(&rqst[i].rq_iter, INT_MAX);
}
nents += DIV_ROUND_UP(offset_in_page(sig) + SMB2_SIGNATURE_SIZE, PAGE_SIZE);
return nents;
@ -2192,9 +2191,9 @@ static inline unsigned int cifs_get_num_sgs(const struct smb_rqst *rqst,
/* We can not use the normal sg_set_buf() as we will sometimes pass a
* stack object as buf.
*/
static inline struct scatterlist *cifs_sg_set_buf(struct scatterlist *sg,
const void *buf,
unsigned int buflen)
static inline void cifs_sg_set_buf(struct sg_table *sgtable,
const void *buf,
unsigned int buflen)
{
unsigned long addr = (unsigned long)buf;
unsigned int off = offset_in_page(addr);
@ -2204,16 +2203,17 @@ static inline struct scatterlist *cifs_sg_set_buf(struct scatterlist *sg,
do {
unsigned int len = min_t(unsigned int, buflen, PAGE_SIZE - off);
sg_set_page(sg++, vmalloc_to_page((void *)addr), len, off);
sg_set_page(&sgtable->sgl[sgtable->nents++],
vmalloc_to_page((void *)addr), len, off);
off = 0;
addr += PAGE_SIZE;
buflen -= len;
} while (buflen);
} else {
sg_set_page(sg++, virt_to_page(addr), buflen, off);
sg_set_page(&sgtable->sgl[sgtable->nents++],
virt_to_page(addr), buflen, off);
}
return sg;
}
#endif /* _CIFS_GLOB_H */

8
fs/cifs/cifsproto.h
View File

@ -584,10 +584,7 @@ int cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid);
int cifs_async_writev(struct cifs_writedata *wdata,
void (*release)(struct kref *kref));
void cifs_writev_complete(struct work_struct *work);
struct cifs_writedata *cifs_writedata_alloc(unsigned int nr_pages,
work_func_t complete);
struct cifs_writedata *cifs_writedata_direct_alloc(struct page **pages,
work_func_t complete);
struct cifs_writedata *cifs_writedata_alloc(work_func_t complete);
void cifs_writedata_release(struct kref *refcount);
int cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon,
struct cifs_sb_info *cifs_sb,
@ -604,13 +601,10 @@ enum securityEnum cifs_select_sectype(struct TCP_Server_Info *,
enum securityEnum);
struct cifs_aio_ctx *cifs_aio_ctx_alloc(void);
void cifs_aio_ctx_release(struct kref *refcount);
int setup_aio_ctx_iter(struct cifs_aio_ctx *ctx, struct iov_iter *iter, int rw);
int cifs_alloc_hash(const char *name, struct shash_desc **sdesc);
void cifs_free_hash(struct shash_desc **sdesc);
void rqst_page_get_length(const struct smb_rqst *rqst, unsigned int page,
unsigned int *len, unsigned int *offset);
struct cifs_chan *
cifs_ses_find_chan(struct cifs_ses *ses, struct TCP_Server_Info *server);
int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses);

15
fs/cifs/cifssmb.c
View File

@ -24,6 +24,7 @@
#include <linux/task_io_accounting_ops.h>
#include <linux/uaccess.h>
#include "cifspdu.h"
#include "cifsfs.h"
#include "cifsglob.h"
#include "cifsacl.h"
#include "cifsproto.h"
@ -1294,11 +1295,8 @@ cifs_readv_callback(struct mid_q_entry *mid)
struct TCP_Server_Info *server = tcon->ses->server;
struct smb_rqst rqst = { .rq_iov = rdata->iov,
.rq_nvec = 2,
.rq_pages = rdata->pages,
.rq_offset = rdata->page_offset,
.rq_npages = rdata->nr_pages,
.rq_pagesz = rdata->pagesz,
.rq_tailsz = rdata->tailsz };
.rq_iter_size = iov_iter_count(&rdata->iter),
.rq_iter = rdata->iter };
struct cifs_credits credits = { .value = 1, .instance = 0 };
cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%u\n",
@ -1737,11 +1735,8 @@ cifs_async_writev(struct cifs_writedata *wdata,
rqst.rq_iov = iov;
rqst.rq_nvec = 2;
rqst.rq_pages = wdata->pages;
rqst.rq_offset = wdata->page_offset;
rqst.rq_npages = wdata->nr_pages;
rqst.rq_pagesz = wdata->pagesz;
rqst.rq_tailsz = wdata->tailsz;
rqst.rq_iter = wdata->iter;
rqst.rq_iter_size = iov_iter_count(&wdata->iter);
cifs_dbg(FYI, "async write at %llu %u bytes\n",
wdata->offset, wdata->bytes);

1197
fs/cifs/file.c
View File

File diff suppressed because it is too large Load Diff

22
fs/cifs/fscache.c
View File

@ -165,22 +165,16 @@ static int fscache_fallback_read_page(struct inode *inode, struct page *page)
/*
* Fallback page writing interface.
*/
static int fscache_fallback_write_page(struct inode *inode, struct page *page,
bool no_space_allocated_yet)
static int fscache_fallback_write_pages(struct inode *inode, loff_t start, size_t len,
bool no_space_allocated_yet)
{
struct netfs_cache_resources cres;
struct fscache_cookie *cookie = cifs_inode_cookie(inode);
struct iov_iter iter;
struct bio_vec bvec[1];
loff_t start = page_offset(page);
size_t len = PAGE_SIZE;
int ret;
memset(&cres, 0, sizeof(cres));
bvec[0].bv_page = page;
bvec[0].bv_offset = 0;
bvec[0].bv_len = PAGE_SIZE;
iov_iter_bvec(&iter, ITER_SOURCE, bvec, ARRAY_SIZE(bvec), PAGE_SIZE);
iov_iter_xarray(&iter, ITER_SOURCE, &inode->i_mapping->i_pages, start, len);
ret = fscache_begin_write_operation(&cres, cookie);
if (ret < 0)
@ -189,7 +183,7 @@ static int fscache_fallback_write_page(struct inode *inode, struct page *page,
ret = cres.ops->prepare_write(&cres, &start, &len, i_size_read(inode),
no_space_allocated_yet);
if (ret == 0)
ret = fscache_write(&cres, page_offset(page), &iter, NULL, NULL);
ret = fscache_write(&cres, start, &iter, NULL, NULL);
fscache_end_operation(&cres);
return ret;
}
@ -213,12 +207,12 @@ int __cifs_readpage_from_fscache(struct inode *inode, struct page *page)
return 0;
}
void __cifs_readpage_to_fscache(struct inode *inode, struct page *page)
void __cifs_readahead_to_fscache(struct inode *inode, loff_t pos, size_t len)
{
cifs_dbg(FYI, "%s: (fsc: %p, p: %p, i: %p)\n",
__func__, cifs_inode_cookie(inode), page, inode);
cifs_dbg(FYI, "%s: (fsc: %p, p: %llx, l: %zx, i: %p)\n",
__func__, cifs_inode_cookie(inode), pos, len, inode);
fscache_fallback_write_page(inode, page, true);
fscache_fallback_write_pages(inode, pos, len, true);
}
/*

10
fs/cifs/fscache.h
View File

@ -90,7 +90,7 @@ static inline int cifs_fscache_query_occupancy(struct inode *inode,
}
extern int __cifs_readpage_from_fscache(struct inode *pinode, struct page *ppage);
extern void __cifs_readpage_to_fscache(struct inode *pinode, struct page *ppage);
extern void __cifs_readahead_to_fscache(struct inode *pinode, loff_t pos, size_t len);
static inline int cifs_readpage_from_fscache(struct inode *inode,
@ -101,11 +101,11 @@ static inline int cifs_readpage_from_fscache(struct inode *inode,
return -ENOBUFS;
}
static inline void cifs_readpage_to_fscache(struct inode *inode,
struct page *page)
static inline void cifs_readahead_to_fscache(struct inode *inode,
loff_t pos, size_t len)
{
if (cifs_inode_cookie(inode))
__cifs_readpage_to_fscache(inode, page);
__cifs_readahead_to_fscache(inode, pos, len);
}
#else /* CONFIG_CIFS_FSCACHE */
@ -141,7 +141,7 @@ cifs_readpage_from_fscache(struct inode *inode, struct page *page)
}
static inline
void cifs_readpage_to_fscache(struct inode *inode, struct page *page) {}
void cifs_readahead_to_fscache(struct inode *inode, loff_t pos, size_t len) {}
#endif /* CONFIG_CIFS_FSCACHE */

128
fs/cifs/misc.c
View File

@ -966,16 +966,22 @@ cifs_aio_ctx_release(struct kref *refcount)
/*
* ctx->bv is only set if setup_aio_ctx_iter() was call successfuly
* which means that iov_iter_get_pages() was a success and thus that
* we have taken reference on pages.
* which means that iov_iter_extract_pages() was a success and thus
* that we may have references or pins on pages that we need to
* release.
*/
if (ctx->bv) {
unsigned i;
if (ctx->should_dirty || ctx->bv_need_unpin) {
unsigned int i;
for (i = 0; i < ctx->npages; i++) {
if (ctx->should_dirty)
set_page_dirty(ctx->bv[i].bv_page);
put_page(ctx->bv[i].bv_page);
for (i = 0; i < ctx->nr_pinned_pages; i++) {
struct page *page = ctx->bv[i].bv_page;
if (ctx->should_dirty)
set_page_dirty(page);
if (ctx->bv_need_unpin)
unpin_user_page(page);
}
}
kvfree(ctx->bv);
}
@ -983,95 +989,6 @@ cifs_aio_ctx_release(struct kref *refcount)
kfree(ctx);
}
#define CIFS_AIO_KMALLOC_LIMIT (1024 * 1024)
int
setup_aio_ctx_iter(struct cifs_aio_ctx *ctx, struct iov_iter *iter, int rw)
{
ssize_t rc;
unsigned int cur_npages;
unsigned int npages = 0;
unsigned int i;
size_t len;
size_t count = iov_iter_count(iter);
unsigned int saved_len;
size_t start;
unsigned int max_pages = iov_iter_npages(iter, INT_MAX);
struct page **pages = NULL;
struct bio_vec *bv = NULL;
if (iov_iter_is_kvec(iter)) {
memcpy(&ctx->iter, iter, sizeof(*iter));
ctx->len = count;
iov_iter_advance(iter, count);
return 0;
}
if (array_size(max_pages, sizeof(*bv)) <= CIFS_AIO_KMALLOC_LIMIT)
bv = kmalloc_array(max_pages, sizeof(*bv), GFP_KERNEL);
if (!bv) {
bv = vmalloc(array_size(max_pages, sizeof(*bv)));
if (!bv)
return -ENOMEM;
}
if (array_size(max_pages, sizeof(*pages)) <= CIFS_AIO_KMALLOC_LIMIT)
pages = kmalloc_array(max_pages, sizeof(*pages), GFP_KERNEL);
if (!pages) {
pages = vmalloc(array_size(max_pages, sizeof(*pages)));
if (!pages) {
kvfree(bv);
return -ENOMEM;
}
}
saved_len = count;
while (count && npages < max_pages) {
rc = iov_iter_get_pages2(iter, pages, count, max_pages, &start);
if (rc < 0) {
cifs_dbg(VFS, "Couldn't get user pages (rc=%zd)\n", rc);
break;
}
if (rc > count) {
cifs_dbg(VFS, "get pages rc=%zd more than %zu\n", rc,
count);
break;
}
count -= rc;
rc += start;
cur_npages = DIV_ROUND_UP(rc, PAGE_SIZE);
if (npages + cur_npages > max_pages) {
cifs_dbg(VFS, "out of vec array capacity (%u vs %u)\n",
npages + cur_npages, max_pages);
break;
}
for (i = 0; i < cur_npages; i++) {
len = rc > PAGE_SIZE ? PAGE_SIZE : rc;
bv[npages + i].bv_page = pages[i];
bv[npages + i].bv_offset = start;
bv[npages + i].bv_len = len - start;
rc -= len;
start = 0;
}
npages += cur_npages;
}
kvfree(pages);
ctx->bv = bv;
ctx->len = saved_len - count;
ctx->npages = npages;
iov_iter_bvec(&ctx->iter, rw, ctx->bv, npages, ctx->len);
return 0;
}
/**
* cifs_alloc_hash - allocate hash and hash context together
* @name: The name of the crypto hash algo
@ -1129,25 +1046,6 @@ cifs_free_hash(struct shash_desc **sdesc)
*sdesc = NULL;
}
/**
* rqst_page_get_length - obtain the length and offset for a page in smb_rqst
* @rqst: The request descriptor
* @page: The index of the page to query
* @len: Where to store the length for this page:
* @offset: Where to store the offset for this page
*/
void rqst_page_get_length(const struct smb_rqst *rqst, unsigned int page,
unsigned int *len, unsigned int *offset)
{
*len = rqst->rq_pagesz;
*offset = (page == 0) ? rqst->rq_offset : 0;
if (rqst->rq_npages == 1 || page == rqst->rq_npages-1)
*len = rqst->rq_tailsz;
else if (page == 0)
*len = rqst->rq_pagesz - rqst->rq_offset;
}
void extract_unc_hostname(const char *unc, const char **h, size_t *len)
{
const char *end;

376
fs/cifs/smb2ops.c
View File

@ -4238,8 +4238,8 @@ fill_transform_hdr(struct smb2_transform_hdr *tr_hdr, unsigned int orig_len,
static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst *rqst,
int num_rqst, const u8 *sig, u8 **iv,
struct aead_request **req, struct scatterlist **sgl,
unsigned int *num_sgs)
struct aead_request **req, struct sg_table *sgt,
unsigned int *num_sgs, size_t *sensitive_size)
{
unsigned int req_size = sizeof(**req) + crypto_aead_reqsize(tfm);
unsigned int iv_size = crypto_aead_ivsize(tfm);
@ -4247,43 +4247,45 @@ static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst
u8 *p;
*num_sgs = cifs_get_num_sgs(rqst, num_rqst, sig);
if (IS_ERR_VALUE((long)(int)*num_sgs))
return ERR_PTR(*num_sgs);
len = iv_size;
len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1);
len = ALIGN(len, crypto_tfm_ctx_alignment());
len += req_size;
len = ALIGN(len, __alignof__(struct scatterlist));
len += *num_sgs * sizeof(**sgl);
len += array_size(*num_sgs, sizeof(struct scatterlist));
*sensitive_size = len;
p = kmalloc(len, GFP_ATOMIC);
p = kvzalloc(len, GFP_NOFS);
if (!p)
return NULL;
return ERR_PTR(-ENOMEM);
*iv = (u8 *)PTR_ALIGN(p, crypto_aead_alignmask(tfm) + 1);
*req = (struct aead_request *)PTR_ALIGN(*iv + iv_size,
crypto_tfm_ctx_alignment());
*sgl = (struct scatterlist *)PTR_ALIGN((u8 *)*req + req_size,
__alignof__(struct scatterlist));
sgt->sgl = (struct scatterlist *)PTR_ALIGN((u8 *)*req + req_size,
__alignof__(struct scatterlist));
return p;
}
static void *smb2_get_aead_req(struct crypto_aead *tfm, const struct smb_rqst *rqst,
static void *smb2_get_aead_req(struct crypto_aead *tfm, struct smb_rqst *rqst,
int num_rqst, const u8 *sig, u8 **iv,
struct aead_request **req, struct scatterlist **sgl)
struct aead_request **req, struct scatterlist **sgl,
size_t *sensitive_size)
{
unsigned int off, len, skip;
struct scatterlist *sg;
unsigned int num_sgs;
unsigned long addr;
int i, j;
struct sg_table sgtable = {};
unsigned int skip, num_sgs, i, j;
ssize_t rc;
void *p;
p = smb2_aead_req_alloc(tfm, rqst, num_rqst, sig, iv, req, sgl, &num_sgs);
if (!p)
return NULL;
p = smb2_aead_req_alloc(tfm, rqst, num_rqst, sig, iv, req, &sgtable,
&num_sgs, sensitive_size);
if (IS_ERR(p))
return ERR_CAST(p);
sg_init_table(*sgl, num_sgs);
sg = *sgl;
sg_init_marker(sgtable.sgl, num_sgs);
/*
* The first rqst has a transform header where the
@ -4291,30 +4293,29 @@ static void *smb2_get_aead_req(struct crypto_aead *tfm, const struct smb_rqst *r
*/
skip = 20;
/* Assumes the first rqst has a transform header as the first iov.
* I.e.
* rqst[0].rq_iov[0] is transform header
* rqst[0].rq_iov[1+] data to be encrypted/decrypted
* rqst[1+].rq_iov[0+] data to be encrypted/decrypted
*/
for (i = 0; i < num_rqst; i++) {
for (j = 0; j < rqst[i].rq_nvec; j++) {
struct kvec *iov = &rqst[i].rq_iov[j];
struct iov_iter *iter = &rqst[i].rq_iter;
size_t count = iov_iter_count(iter);
addr = (unsigned long)iov->iov_base + skip;
len = iov->iov_len - skip;
sg = cifs_sg_set_buf(sg, (void *)addr, len);
for (j = 0; j < rqst[i].rq_nvec; j++) {
cifs_sg_set_buf(&sgtable,
rqst[i].rq_iov[j].iov_base + skip,
rqst[i].rq_iov[j].iov_len - skip);
/* See the above comment on the 'skip' assignment */
skip = 0;
}
for (j = 0; j < rqst[i].rq_npages; j++) {
rqst_page_get_length(&rqst[i], j, &len, &off);
sg_set_page(sg++, rqst[i].rq_pages[j], len, off);
}
}
cifs_sg_set_buf(sg, sig, SMB2_SIGNATURE_SIZE);
sgtable.orig_nents = sgtable.nents;
rc = netfs_extract_iter_to_sg(iter, count, &sgtable,
num_sgs - sgtable.nents, 0);
iov_iter_revert(iter, rc);
sgtable.orig_nents = sgtable.nents;
}
cifs_sg_set_buf(&sgtable, sig, SMB2_SIGNATURE_SIZE);
sg_mark_end(&sgtable.sgl[sgtable.nents - 1]);
*sgl = sgtable.sgl;
return p;
}
@ -4368,6 +4369,7 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst,
struct crypto_aead *tfm;
unsigned int crypt_len = le32_to_cpu(tr_hdr->OriginalMessageSize);
void *creq;
size_t sensitive_size;
rc = smb2_get_enc_key(server, le64_to_cpu(tr_hdr->SessionId), enc, key);
if (rc) {
@ -4401,9 +4403,10 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst,
return rc;
}
creq = smb2_get_aead_req(tfm, rqst, num_rqst, sign, &iv, &req, &sg);
if (unlikely(!creq))
return -ENOMEM;
creq = smb2_get_aead_req(tfm, rqst, num_rqst, sign, &iv, &req, &sg,
&sensitive_size);
if (IS_ERR(creq))
return PTR_ERR(creq);
if (!enc) {
memcpy(sign, &tr_hdr->Signature, SMB2_SIGNATURE_SIZE);
@ -4431,22 +4434,35 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst,
if (!rc && enc)
memcpy(&tr_hdr->Signature, sign, SMB2_SIGNATURE_SIZE);
kfree_sensitive(creq);
kvfree_sensitive(creq, sensitive_size);
return rc;
}
/*
* Clear a read buffer, discarding the folios which have XA_MARK_0 set.
*/
static void cifs_clear_xarray_buffer(struct xarray *buffer)
{
struct folio *folio;
XA_STATE(xas, buffer, 0);
rcu_read_lock();
xas_for_each_marked(&xas, folio, ULONG_MAX, XA_MARK_0) {
folio_put(folio);
}
rcu_read_unlock();
xa_destroy(buffer);
}
void
smb3_free_compound_rqst(int num_rqst, struct smb_rqst *rqst)
{
int i, j;
int i;
for (i = 0; i < num_rqst; i++) {
if (rqst[i].rq_pages) {
for (j = rqst[i].rq_npages - 1; j >= 0; j--)
put_page(rqst[i].rq_pages[j]);
kfree(rqst[i].rq_pages);
}
}
for (i = 0; i < num_rqst; i++)
if (!xa_empty(&rqst[i].rq_buffer))
cifs_clear_xarray_buffer(&rqst[i].rq_buffer);
}
/*
@ -4466,9 +4482,8 @@ static int
smb3_init_transform_rq(struct TCP_Server_Info *server, int num_rqst,
struct smb_rqst *new_rq, struct smb_rqst *old_rq)
{
struct page **pages;
struct smb2_transform_hdr *tr_hdr = new_rq[0].rq_iov[0].iov_base;
unsigned int npages;
struct page *page;
unsigned int orig_len = 0;
int i, j;
int rc = -ENOMEM;
@ -4476,40 +4491,45 @@ smb3_init_transform_rq(struct TCP_Server_Info *server, int num_rqst,
for (i = 1; i < num_rqst; i++) {
struct smb_rqst *old = &old_rq[i - 1];
struct smb_rqst *new = &new_rq[i];
struct xarray *buffer = &new->rq_buffer;
size_t size = iov_iter_count(&old->rq_iter), seg, copied = 0;
orig_len += smb_rqst_len(server, old);
new->rq_iov = old->rq_iov;
new->rq_nvec = old->rq_nvec;
npages = old->rq_npages;
if (!npages)
continue;
xa_init(buffer);
pages = kmalloc_array(npages, sizeof(struct page *),
GFP_KERNEL);
if (!pages)
goto err_free;
if (size > 0) {
unsigned int npages = DIV_ROUND_UP(size, PAGE_SIZE);
new->rq_pages = pages;
new->rq_npages = npages;
new->rq_offset = old->rq_offset;
new->rq_pagesz = old->rq_pagesz;
new->rq_tailsz = old->rq_tailsz;
for (j = 0; j < npages; j++) {
void *o;
for (j = 0; j < npages; j++) {
pages[j] = alloc_page(GFP_KERNEL|__GFP_HIGHMEM);
if (!pages[j])
goto err_free;
}
rc = -ENOMEM;
page = alloc_page(GFP_KERNEL|__GFP_HIGHMEM);
if (!page)
goto err_free;
page->index = j;
o = xa_store(buffer, j, page, GFP_KERNEL);
if (xa_is_err(o)) {
rc = xa_err(o);
put_page(page);
goto err_free;
}
/* copy pages form the old */
for (j = 0; j < npages; j++) {
unsigned int offset, len;
xa_set_mark(buffer, j, XA_MARK_0);
rqst_page_get_length(new, j, &len, &offset);
memcpy_page(new->rq_pages[j], offset,
old->rq_pages[j], offset, len);
seg = min_t(size_t, size - copied, PAGE_SIZE);
if (copy_page_from_iter(page, 0, seg, &old->rq_iter) != seg) {
rc = -EFAULT;
goto err_free;
}
copied += seg;
}
iov_iter_xarray(&new->rq_iter, ITER_SOURCE,
buffer, 0, size);
new->rq_iter_size = size;
}
}
@ -4538,12 +4558,12 @@ smb3_is_transform_hdr(void *buf)
static int
decrypt_raw_data(struct TCP_Server_Info *server, char *buf,
unsigned int buf_data_size, struct page **pages,
unsigned int npages, unsigned int page_data_size,
unsigned int buf_data_size, struct iov_iter *iter,
bool is_offloaded)
{
struct kvec iov[2];
struct smb_rqst rqst = {NULL};
size_t iter_size = 0;
int rc;
iov[0].iov_base = buf;
@ -4553,10 +4573,11 @@ decrypt_raw_data(struct TCP_Server_Info *server, char *buf,
rqst.rq_iov = iov;
rqst.rq_nvec = 2;
rqst.rq_pages = pages;
rqst.rq_npages = npages;
rqst.rq_pagesz = PAGE_SIZE;
rqst.rq_tailsz = (page_data_size % PAGE_SIZE) ? : PAGE_SIZE;
if (iter) {
rqst.rq_iter = *iter;
rqst.rq_iter_size = iov_iter_count(iter);
iter_size = iov_iter_count(iter);
}
rc = crypt_message(server, 1, &rqst, 0);
cifs_dbg(FYI, "Decrypt message returned %d\n", rc);
@ -4567,73 +4588,37 @@ decrypt_raw_data(struct TCP_Server_Info *server, char *buf,
memmove(buf, iov[1].iov_base, buf_data_size);
if (!is_offloaded)
server->total_read = buf_data_size + page_data_size;
server->total_read = buf_data_size + iter_size;
return rc;
}
static int
read_data_into_pages(struct TCP_Server_Info *server, struct page **pages,
unsigned int npages, unsigned int len)
cifs_copy_pages_to_iter(struct xarray *pages, unsigned int data_size,
unsigned int skip, struct iov_iter *iter)
{
int i;
int length;
struct page *page;
unsigned long index;
for (i = 0; i < npages; i++) {
struct page *page = pages[i];
size_t n;
xa_for_each(pages, index, page) {
size_t n, len = min_t(unsigned int, PAGE_SIZE - skip, data_size);
n = len;
if (len >= PAGE_SIZE) {
/* enough data to fill the page */
n = PAGE_SIZE;
len -= n;
} else {
zero_user(page, len, PAGE_SIZE - len);
len = 0;
n = copy_page_to_iter(page, skip, len, iter);
if (n != len) {
cifs_dbg(VFS, "%s: something went wrong\n", __func__);
return -EIO;
}
length = cifs_read_page_from_socket(server, page, 0, n);
if (length < 0)
return length;
server->total_read += length;
data_size -= n;
skip = 0;
}
return 0;
}
static int
init_read_bvec(struct page **pages, unsigned int npages, unsigned int data_size,
unsigned int cur_off, struct bio_vec **page_vec)
{
struct bio_vec *bvec;
int i;
bvec = kcalloc(npages, sizeof(struct bio_vec), GFP_KERNEL);
if (!bvec)
return -ENOMEM;
for (i = 0; i < npages; i++) {
bvec[i].bv_page = pages[i];
bvec[i].bv_offset = (i == 0) ? cur_off : 0;
bvec[i].bv_len = min_t(unsigned int, PAGE_SIZE, data_size);
data_size -= bvec[i].bv_len;
}
if (data_size != 0) {
cifs_dbg(VFS, "%s: something went wrong\n", __func__);
kfree(bvec);
return -EIO;
}
*page_vec = bvec;
return 0;
}
static int
handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
char *buf, unsigned int buf_len, struct page **pages,
unsigned int npages, unsigned int page_data_size,
bool is_offloaded)
char *buf, unsigned int buf_len, struct xarray *pages,
unsigned int pages_len, bool is_offloaded)
{
unsigned int data_offset;
unsigned int data_len;
@ -4642,9 +4627,6 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
unsigned int pad_len;
struct cifs_readdata *rdata = mid->callback_data;
struct smb2_hdr *shdr = (struct smb2_hdr *)buf;
struct bio_vec *bvec = NULL;
struct iov_iter iter;
struct kvec iov;
int length;
bool use_rdma_mr = false;
@ -4733,7 +4715,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
return 0;
}
if (data_len > page_data_size - pad_len) {
if (data_len > pages_len - pad_len) {
/* data_len is corrupt -- discard frame */
rdata->result = -EIO;
if (is_offloaded)
@ -4743,8 +4725,9 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
return 0;
}
rdata->result = init_read_bvec(pages, npages, page_data_size,
cur_off, &bvec);
/* Copy the data to the output I/O iterator. */
rdata->result = cifs_copy_pages_to_iter(pages, pages_len,
cur_off, &rdata->iter);
if (rdata->result != 0) {
if (is_offloaded)
mid->mid_state = MID_RESPONSE_MALFORMED;
@ -4752,14 +4735,16 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
dequeue_mid(mid, rdata->result);
return 0;
}
rdata->got_bytes = pages_len;
iov_iter_bvec(&iter, ITER_SOURCE, bvec, npages, data_len);
} else if (buf_len >= data_offset + data_len) {
/* read response payload is in buf */
WARN_ONCE(npages > 0, "read data can be either in buf or in pages");
iov.iov_base = buf + data_offset;
iov.iov_len = data_len;
iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, data_len);
WARN_ONCE(pages && !xa_empty(pages),
"read data can be either in buf or in pages");
length = copy_to_iter(buf + data_offset, data_len, &rdata->iter);
if (length < 0)
return length;
rdata->got_bytes = data_len;
} else {
/* read response payload cannot be in both buf and pages */
WARN_ONCE(1, "buf can not contain only a part of read data");
@ -4771,26 +4756,18 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
return 0;
}
length = rdata->copy_into_pages(server, rdata, &iter);
kfree(bvec);
if (length < 0)
return length;
if (is_offloaded)
mid->mid_state = MID_RESPONSE_RECEIVED;
else
dequeue_mid(mid, false);
return length;
return 0;
}
struct smb2_decrypt_work {
struct work_struct decrypt;
struct TCP_Server_Info *server;
struct page **ppages;
struct xarray buffer;
char *buf;
unsigned int npages;
unsigned int len;
};
@ -4799,11 +4776,13 @@ static void smb2_decrypt_offload(struct work_struct *work)
{
struct smb2_decrypt_work *dw = container_of(work,
struct smb2_decrypt_work, decrypt);
int i, rc;
int rc;
struct mid_q_entry *mid;
struct iov_iter iter;
iov_iter_xarray(&iter, ITER_DEST, &dw->buffer, 0, dw->len);
rc = decrypt_raw_data(dw->server, dw->buf, dw->server->vals->read_rsp_size,
dw->ppages, dw->npages, dw->len, true);
&iter, true);
if (rc) {
cifs_dbg(VFS, "error decrypting rc=%d\n", rc);
goto free_pages;
@ -4817,7 +4796,7 @@ static void smb2_decrypt_offload(struct work_struct *work)
mid->decrypted = true;
rc = handle_read_data(dw->server, mid, dw->buf,
dw->server->vals->read_rsp_size,
dw->ppages, dw->npages, dw->len,
&dw->buffer, dw->len,
true);
if (rc >= 0) {
#ifdef CONFIG_CIFS_STATS2
@ -4850,10 +4829,7 @@ static void smb2_decrypt_offload(struct work_struct *work)
}
free_pages:
for (i = dw->npages-1; i >= 0; i--)
put_page(dw->ppages[i]);
kfree(dw->ppages);
cifs_clear_xarray_buffer(&dw->buffer);
cifs_small_buf_release(dw->buf);
kfree(dw);
}
@ -4863,47 +4839,66 @@ static int
receive_encrypted_read(struct TCP_Server_Info *server, struct mid_q_entry **mid,
int *num_mids)
{
struct page *page;
char *buf = server->smallbuf;
struct smb2_transform_hdr *tr_hdr = (struct smb2_transform_hdr *)buf;
unsigned int npages;
struct page **pages;
unsigned int len;
struct iov_iter iter;
unsigned int len, npages;
unsigned int buflen = server->pdu_size;
int rc;
int i = 0;
struct smb2_decrypt_work *dw;
dw = kzalloc(sizeof(struct smb2_decrypt_work), GFP_KERNEL);
if (!dw)
return -ENOMEM;
xa_init(&dw->buffer);
INIT_WORK(&dw->decrypt, smb2_decrypt_offload);
dw->server = server;
*num_mids = 1;
len = min_t(unsigned int, buflen, server->vals->read_rsp_size +
sizeof(struct smb2_transform_hdr)) - HEADER_SIZE(server) + 1;
rc = cifs_read_from_socket(server, buf + HEADER_SIZE(server) - 1, len);
if (rc < 0)
return rc;
goto free_dw;
server->total_read += rc;
len = le32_to_cpu(tr_hdr->OriginalMessageSize) -
server->vals->read_rsp_size;
dw->len = len;
npages = DIV_ROUND_UP(len, PAGE_SIZE);
pages = kmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
if (!pages) {
rc = -ENOMEM;
goto discard_data;
}
rc = -ENOMEM;
for (; i < npages; i++) {
pages[i] = alloc_page(GFP_KERNEL|__GFP_HIGHMEM);
if (!pages[i]) {
rc = -ENOMEM;
void *old;
page = alloc_page(GFP_KERNEL|__GFP_HIGHMEM);
if (!page)
goto discard_data;
page->index = i;
old = xa_store(&dw->buffer, i, page, GFP_KERNEL);
if (xa_is_err(old)) {
rc = xa_err(old);
put_page(page);
goto discard_data;
}
xa_set_mark(&dw->buffer, i, XA_MARK_0);
}
/* read read data into pages */
rc = read_data_into_pages(server, pages, npages, len);
if (rc)
goto free_pages;
iov_iter_xarray(&iter, ITER_DEST, &dw->buffer, 0, npages * PAGE_SIZE);
/* Read the data into the buffer and clear excess bufferage. */
rc = cifs_read_iter_from_socket(server, &iter, dw->len);
if (rc < 0)
goto discard_data;
server->total_read += rc;
if (rc < npages * PAGE_SIZE)
iov_iter_zero(npages * PAGE_SIZE - rc, &iter);
iov_iter_revert(&iter, npages * PAGE_SIZE);
iov_iter_truncate(&iter, dw->len);
rc = cifs_discard_remaining_data(server);
if (rc)
@ -4916,39 +4911,28 @@ receive_encrypted_read(struct TCP_Server_Info *server, struct mid_q_entry **mid,
if ((server->min_offload) && (server->in_flight > 1) &&
(server->pdu_size >= server->min_offload)) {
dw = kmalloc(sizeof(struct smb2_decrypt_work), GFP_KERNEL);
if (dw == NULL)
goto non_offloaded_decrypt;
dw->buf = server->smallbuf;
server->smallbuf = (char *)cifs_small_buf_get();
INIT_WORK(&dw->decrypt, smb2_decrypt_offload);
dw->npages = npages;
dw->server = server;
dw->ppages = pages;
dw->len = len;
queue_work(decrypt_wq, &dw->decrypt);
*num_mids = 0; /* worker thread takes care of finding mid */
return -1;
}
non_offloaded_decrypt:
rc = decrypt_raw_data(server, buf, server->vals->read_rsp_size,
pages, npages, len, false);
&iter, false);
if (rc)
goto free_pages;
*mid = smb2_find_mid(server, buf);
if (*mid == NULL)
if (*mid == NULL) {
cifs_dbg(FYI, "mid not found\n");
else {
} else {
cifs_dbg(FYI, "mid found\n");
(*mid)->decrypted = true;
rc = handle_read_data(server, *mid, buf,
server->vals->read_rsp_size,
pages, npages, len, false);
&dw->buffer, dw->len, false);
if (rc >= 0) {
if (server->ops->is_network_name_deleted) {
server->ops->is_network_name_deleted(buf,
@ -4958,9 +4942,9 @@ non_offloaded_decrypt:
}
free_pages:
for (i = i - 1; i >= 0; i--)
put_page(pages[i]);
kfree(pages);
cifs_clear_xarray_buffer(&dw->buffer);
free_dw:
kfree(dw);
return rc;
discard_data:
cifs_discard_remaining_data(server);
@ -4998,7 +4982,7 @@ receive_encrypted_standard(struct TCP_Server_Info *server,
server->total_read += length;
buf_size = pdu_length - sizeof(struct smb2_transform_hdr);
length = decrypt_raw_data(server, buf, buf_size, NULL, 0, 0, false);
length = decrypt_raw_data(server, buf, buf_size, NULL, false);
if (length)
return length;
@ -5097,7 +5081,7 @@ smb3_handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid)
char *buf = server->large_buf ? server->bigbuf : server->smallbuf;
return handle_read_data(server, mid, buf, server->pdu_size,
NULL, 0, 0, false);
NULL, 0, false);
}
static int

53
fs/cifs/smb2pdu.c
View File

@ -4139,10 +4139,8 @@ smb2_new_read_req(void **buf, unsigned int *total_len,
struct smbd_buffer_descriptor_v1 *v1;
bool need_invalidate = server->dialect == SMB30_PROT_ID;
rdata->mr = smbd_register_mr(
server->smbd_conn, rdata->pages,
rdata->nr_pages, rdata->page_offset,
rdata->tailsz, true, need_invalidate);
rdata->mr = smbd_register_mr(server->smbd_conn, &rdata->iter,
true, need_invalidate);
if (!rdata->mr)
return -EAGAIN;
@ -4199,15 +4197,9 @@ smb2_readv_callback(struct mid_q_entry *mid)
(struct smb2_hdr *)rdata->iov[0].iov_base;
struct cifs_credits credits = { .value = 0, .instance = 0 };
struct smb_rqst rqst = { .rq_iov = &rdata->iov[1],
.rq_nvec = 1, };
if (rdata->got_bytes) {
rqst.rq_pages = rdata->pages;
rqst.rq_offset = rdata->page_offset;
rqst.rq_npages = rdata->nr_pages;
rqst.rq_pagesz = rdata->pagesz;
rqst.rq_tailsz = rdata->tailsz;
}
.rq_nvec = 1,
.rq_iter = rdata->iter,
.rq_iter_size = iov_iter_count(&rdata->iter), };
WARN_ONCE(rdata->server != mid->server,
"rdata server %p != mid server %p",
@ -4225,6 +4217,8 @@ smb2_readv_callback(struct mid_q_entry *mid)
if (server->sign && !mid->decrypted) {
int rc;
iov_iter_revert(&rqst.rq_iter, rdata->got_bytes);
iov_iter_truncate(&rqst.rq_iter, rdata->got_bytes);
rc = smb2_verify_signature(&rqst, server);
if (rc)
cifs_tcon_dbg(VFS, "SMB signature verification returned error = %d\n",
@ -4567,7 +4561,7 @@ smb2_async_writev(struct cifs_writedata *wdata,
req->VolatileFileId = io_parms->volatile_fid;
req->WriteChannelInfoOffset = 0;
req->WriteChannelInfoLength = 0;
req->Channel = 0;
req->Channel = SMB2_CHANNEL_NONE;
req->Offset = cpu_to_le64(io_parms->offset);
req->DataOffset = cpu_to_le16(
offsetof(struct smb2_write_req, Buffer));
@ -4587,26 +4581,18 @@ smb2_async_writev(struct cifs_writedata *wdata,
*/
if (smb3_use_rdma_offload(io_parms)) {
struct smbd_buffer_descriptor_v1 *v1;
size_t data_size = iov_iter_count(&wdata->iter);
bool need_invalidate = server->dialect == SMB30_PROT_ID;
wdata->mr = smbd_register_mr(
server->smbd_conn, wdata->pages,
wdata->nr_pages, wdata->page_offset,
wdata->tailsz, false, need_invalidate);
wdata->mr = smbd_register_mr(server->smbd_conn, &wdata->iter,
false, need_invalidate);
if (!wdata->mr) {
rc = -EAGAIN;
goto async_writev_out;
}
req->Length = 0;
req->DataOffset = 0;
if (wdata->nr_pages > 1)
req->RemainingBytes =
cpu_to_le32(
(wdata->nr_pages - 1) * wdata->pagesz -
wdata->page_offset + wdata->tailsz
);
else
req->RemainingBytes = cpu_to_le32(wdata->tailsz);
req->RemainingBytes = cpu_to_le32(data_size);
req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE;
if (need_invalidate)
req->Channel = SMB2_CHANNEL_RDMA_V1;
@ -4625,19 +4611,14 @@ smb2_async_writev(struct cifs_writedata *wdata,
rqst.rq_iov = iov;
rqst.rq_nvec = 1;
rqst.rq_pages = wdata->pages;
rqst.rq_offset = wdata->page_offset;
rqst.rq_npages = wdata->nr_pages;
rqst.rq_pagesz = wdata->pagesz;
rqst.rq_tailsz = wdata->tailsz;
rqst.rq_iter = wdata->iter;
rqst.rq_iter_size = iov_iter_count(&rqst.rq_iter);
#ifdef CONFIG_CIFS_SMB_DIRECT
if (wdata->mr) {
if (wdata->mr)
iov[0].iov_len += sizeof(struct smbd_buffer_descriptor_v1);
rqst.rq_npages = 0;
}
#endif
cifs_dbg(FYI, "async write at %llu %u bytes\n",
io_parms->offset, io_parms->length);
cifs_dbg(FYI, "async write at %llu %u bytes iter=%zx\n",
io_parms->offset, io_parms->length, iov_iter_count(&rqst.rq_iter));
#ifdef CONFIG_CIFS_SMB_DIRECT
/* For RDMA read, I/O size is in RemainingBytes not in Length */

264
fs/cifs/smbdirect.c
View File

@ -34,12 +34,6 @@ static int smbd_post_recv(
struct smbd_response *response);
static int smbd_post_send_empty(struct smbd_connection *info);
static int smbd_post_send_data(
struct smbd_connection *info,
struct kvec *iov, int n_vec, int remaining_data_length);
static int smbd_post_send_page(struct smbd_connection *info,
struct page *page, unsigned long offset,
size_t size, int remaining_data_length);
static void destroy_mr_list(struct smbd_connection *info);
static int allocate_mr_list(struct smbd_connection *info);
@ -986,24 +980,6 @@ err_wait_credit:
return rc;
}
/*
* Send a page
* page: the page to send
* offset: offset in the page to send
* size: length in the page to send
* remaining_data_length: remaining data to send in this payload
*/
static int smbd_post_send_page(struct smbd_connection *info, struct page *page,
unsigned long offset, size_t size, int remaining_data_length)
{
struct scatterlist sgl;
sg_init_table(&sgl, 1);
sg_set_page(&sgl, page, size, offset);
return smbd_post_send_sgl(info, &sgl, size, remaining_data_length);
}
/*
* Send an empty message
* Empty message is used to extend credits to peer to for keep live
@ -1015,35 +991,6 @@ static int smbd_post_send_empty(struct smbd_connection *info)
return smbd_post_send_sgl(info, NULL, 0, 0);
}
/*
* Send a data buffer
* iov: the iov array describing the data buffers
* n_vec: number of iov array
* remaining_data_length: remaining data to send following this packet
* in segmented SMBD packet
*/
static int smbd_post_send_data(
struct smbd_connection *info, struct kvec *iov, int n_vec,
int remaining_data_length)
{
int i;
u32 data_length = 0;
struct scatterlist sgl[SMBDIRECT_MAX_SEND_SGE - 1];
if (n_vec > SMBDIRECT_MAX_SEND_SGE - 1) {
cifs_dbg(VFS, "Can't fit data to SGL, n_vec=%d\n", n_vec);
return -EINVAL;
}
sg_init_table(sgl, n_vec);
for (i = 0; i < n_vec; i++) {
data_length += iov[i].iov_len;
sg_set_buf(&sgl[i], iov[i].iov_base, iov[i].iov_len);
}
return smbd_post_send_sgl(info, sgl, data_length, remaining_data_length);
}
/*
* Post a receive request to the transport
* The remote peer can only send data when a receive request is posted
@ -1987,6 +1934,42 @@ out:
return rc;
}
/*
* Send the contents of an iterator
* @iter: The iterator to send
* @_remaining_data_length: remaining data to send in this payload
*/
static int smbd_post_send_iter(struct smbd_connection *info,
struct iov_iter *iter,
int *_remaining_data_length)
{
struct scatterlist sgl[SMBDIRECT_MAX_SEND_SGE - 1];
unsigned int max_payload = info->max_send_size - sizeof(struct smbd_data_transfer);
ssize_t rc;
/* We're not expecting a user-backed iter */
WARN_ON(iov_iter_extract_will_pin(iter));
do {
struct sg_table sgtable = { .sgl = sgl };
size_t maxlen = min_t(size_t, *_remaining_data_length, max_payload);
sg_init_table(sgtable.sgl, ARRAY_SIZE(sgl));
rc = netfs_extract_iter_to_sg(iter, maxlen,
&sgtable, ARRAY_SIZE(sgl), 0);
if (rc < 0)
break;
if (WARN_ON_ONCE(sgtable.nents == 0))
return -EIO;
sg_mark_end(&sgl[sgtable.nents - 1]);
*_remaining_data_length -= rc;
rc = smbd_post_send_sgl(info, sgl, rc, *_remaining_data_length);
} while (rc == 0 && iov_iter_count(iter) > 0);
return rc;
}
/*
* Send data to transport
* Each rqst is transported as a SMBDirect payload
@ -1997,18 +1980,10 @@ int smbd_send(struct TCP_Server_Info *server,
int num_rqst, struct smb_rqst *rqst_array)
{
struct smbd_connection *info = server->smbd_conn;
struct kvec vecs[SMBDIRECT_MAX_SEND_SGE - 1];
int nvecs;
int size;
unsigned int buflen, remaining_data_length;
unsigned int offset, remaining_vec_data_length;
int start, i, j;
int max_iov_size =
info->max_send_size - sizeof(struct smbd_data_transfer);
struct kvec *iov;
int rc;
struct smb_rqst *rqst;
int rqst_idx;
struct iov_iter iter;
unsigned int remaining_data_length, klen;
int rc, i, rqst_idx;
if (info->transport_status != SMBD_CONNECTED)
return -EAGAIN;
@ -2035,84 +2010,36 @@ int smbd_send(struct TCP_Server_Info *server,
rqst_idx = 0;
do {
rqst = &rqst_array[rqst_idx];
iov = rqst->rq_iov;
cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n",
rqst_idx, smb_rqst_len(server, rqst));
remaining_vec_data_length = 0;
for (i = 0; i < rqst->rq_nvec; i++) {
remaining_vec_data_length += iov[i].iov_len;
dump_smb(iov[i].iov_base, iov[i].iov_len);
rqst_idx, smb_rqst_len(server, rqst));
for (i = 0; i < rqst->rq_nvec; i++)
dump_smb(rqst->rq_iov[i].iov_base, rqst->rq_iov[i].iov_len);
log_write(INFO, "RDMA-WR[%u] nvec=%d len=%u iter=%zu rqlen=%lu\n",
rqst_idx, rqst->rq_nvec, remaining_data_length,
iov_iter_count(&rqst->rq_iter), smb_rqst_len(server, rqst));
/* Send the metadata pages. */
klen = 0;
for (i = 0; i < rqst->rq_nvec; i++)
klen += rqst->rq_iov[i].iov_len;
iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen);
rc = smbd_post_send_iter(info, &iter, &remaining_data_length);
if (rc < 0)
break;
if (iov_iter_count(&rqst->rq_iter) > 0) {
/* And then the data pages if there are any */
rc = smbd_post_send_iter(info, &rqst->rq_iter,
&remaining_data_length);
if (rc < 0)
break;
}
log_write(INFO, "rqst_idx=%d nvec=%d rqst->rq_npages=%d rq_pagesz=%d rq_tailsz=%d buflen=%lu\n",
rqst_idx, rqst->rq_nvec,
rqst->rq_npages, rqst->rq_pagesz,
rqst->rq_tailsz, smb_rqst_len(server, rqst));
start = 0;
offset = 0;
do {
buflen = 0;
i = start;
j = 0;
while (i < rqst->rq_nvec &&
j < SMBDIRECT_MAX_SEND_SGE - 1 &&
buflen < max_iov_size) {
vecs[j].iov_base = iov[i].iov_base + offset;
if (buflen + iov[i].iov_len > max_iov_size) {
vecs[j].iov_len =
max_iov_size - iov[i].iov_len;
buflen = max_iov_size;
offset = vecs[j].iov_len;
} else {
vecs[j].iov_len =
iov[i].iov_len - offset;
buflen += vecs[j].iov_len;
offset = 0;
++i;
}
++j;
}
remaining_vec_data_length -= buflen;
remaining_data_length -= buflen;
log_write(INFO, "sending %s iov[%d] from start=%d nvecs=%d remaining_data_length=%d\n",
remaining_vec_data_length > 0 ?
"partial" : "complete",
rqst->rq_nvec, start, j,
remaining_data_length);
start = i;
rc = smbd_post_send_data(info, vecs, j, remaining_data_length);
if (rc)
goto done;
} while (remaining_vec_data_length > 0);
/* now sending pages if there are any */
for (i = 0; i < rqst->rq_npages; i++) {
rqst_page_get_length(rqst, i, &buflen, &offset);
nvecs = (buflen + max_iov_size - 1) / max_iov_size;
log_write(INFO, "sending pages buflen=%d nvecs=%d\n",
buflen, nvecs);
for (j = 0; j < nvecs; j++) {
size = min_t(unsigned int, max_iov_size, remaining_data_length);
remaining_data_length -= size;
log_write(INFO, "sending pages i=%d offset=%d size=%d remaining_data_length=%d\n",
i, j * max_iov_size + offset, size,
remaining_data_length);
rc = smbd_post_send_page(
info, rqst->rq_pages[i],
j*max_iov_size + offset,
size, remaining_data_length);
if (rc)
goto done;
}
}
} while (++rqst_idx < num_rqst);
done:
/*
* As an optimization, we don't wait for individual I/O to finish
* before sending the next one.
@ -2317,27 +2244,48 @@ again:
goto again;
}
/*
* Transcribe the pages from an iterator into an MR scatterlist.
* @iter: The iterator to transcribe
* @_remaining_data_length: remaining data to send in this payload
*/
static int smbd_iter_to_mr(struct smbd_connection *info,
struct iov_iter *iter,
struct scatterlist *sgl,
unsigned int num_pages)
{
struct sg_table sgtable = { .sgl = sgl };
int ret;
sg_init_table(sgl, num_pages);
ret = netfs_extract_iter_to_sg(iter, iov_iter_count(iter),
&sgtable, num_pages, 0);
WARN_ON(ret < 0);
return ret;
}
/*
* Register memory for RDMA read/write
* pages[]: the list of pages to register memory with
* num_pages: the number of pages to register
* tailsz: if non-zero, the bytes to register in the last page
* iter: the buffer to register memory with
* writing: true if this is a RDMA write (SMB read), false for RDMA read
* need_invalidate: true if this MR needs to be locally invalidated after I/O
* return value: the MR registered, NULL if failed.
*/
struct smbd_mr *smbd_register_mr(
struct smbd_connection *info, struct page *pages[], int num_pages,
int offset, int tailsz, bool writing, bool need_invalidate)
struct smbd_mr *smbd_register_mr(struct smbd_connection *info,
struct iov_iter *iter,
bool writing, bool need_invalidate)
{
struct smbd_mr *smbdirect_mr;
int rc, i;
int rc, num_pages;
enum dma_data_direction dir;
struct ib_reg_wr *reg_wr;
num_pages = iov_iter_npages(iter, info->max_frmr_depth + 1);
if (num_pages > info->max_frmr_depth) {
log_rdma_mr(ERR, "num_pages=%d max_frmr_depth=%d\n",
num_pages, info->max_frmr_depth);
WARN_ON_ONCE(1);
return NULL;
}
@ -2346,32 +2294,16 @@ struct smbd_mr *smbd_register_mr(
log_rdma_mr(ERR, "get_mr returning NULL\n");
return NULL;
}
smbdirect_mr->need_invalidate = need_invalidate;
smbdirect_mr->sgl_count = num_pages;
sg_init_table(smbdirect_mr->sgl, num_pages);
log_rdma_mr(INFO, "num_pages=0x%x offset=0x%x tailsz=0x%x\n",
num_pages, offset, tailsz);
if (num_pages == 1) {
sg_set_page(&smbdirect_mr->sgl[0], pages[0], tailsz, offset);
goto skip_multiple_pages;
}
/* We have at least two pages to register */
sg_set_page(
&smbdirect_mr->sgl[0], pages[0], PAGE_SIZE - offset, offset);
i = 1;
while (i < num_pages - 1) {
sg_set_page(&smbdirect_mr->sgl[i], pages[i], PAGE_SIZE, 0);
i++;
}
sg_set_page(&smbdirect_mr->sgl[i], pages[i],
tailsz ? tailsz : PAGE_SIZE, 0);
skip_multiple_pages:
dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
smbdirect_mr->dir = dir;
smbdirect_mr->need_invalidate = need_invalidate;
smbdirect_mr->sgl_count = num_pages;
log_rdma_mr(INFO, "num_pages=0x%x count=0x%zx\n",
num_pages, iov_iter_count(iter));
smbd_iter_to_mr(info, iter, smbdirect_mr->sgl, num_pages);
rc = ib_dma_map_sg(info->id->device, smbdirect_mr->sgl, num_pages, dir);
if (!rc) {
log_rdma_mr(ERR, "ib_dma_map_sg num_pages=%x dir=%x rc=%x\n",

4
fs/cifs/smbdirect.h
View File

@ -302,8 +302,8 @@ struct smbd_mr {
/* Interfaces to register and deregister MR for RDMA read/write */
struct smbd_mr *smbd_register_mr(
struct smbd_connection *info, struct page *pages[], int num_pages,
int offset, int tailsz, bool writing, bool need_invalidate);
struct smbd_connection *info, struct iov_iter *iter,
bool writing, bool need_invalidate);
int smbd_deregister_mr(struct smbd_mr *mr);
#else

54
fs/cifs/transport.c
View File

@ -270,26 +270,7 @@ smb_rqst_len(struct TCP_Server_Info *server, struct smb_rqst *rqst)
for (i = 0; i < nvec; i++)
buflen += iov[i].iov_len;
/*
* Add in the page array if there is one. The caller needs to make
* sure rq_offset and rq_tailsz are set correctly. If a buffer of
* multiple pages ends at page boundary, rq_tailsz needs to be set to
* PAGE_SIZE.
*/
if (rqst->rq_npages) {
if (rqst->rq_npages == 1)
buflen += rqst->rq_tailsz;
else {
/*
* If there is more than one page, calculate the
* buffer length based on rq_offset and rq_tailsz
*/
buflen += rqst->rq_pagesz * (rqst->rq_npages - 1) -
rqst->rq_offset;
buflen += rqst->rq_tailsz;
}
}
buflen += iov_iter_count(&rqst->rq_iter);
return buflen;
}
@ -376,23 +357,15 @@ __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst,
total_len += sent;
/* now walk the page array and send each page in it */
for (i = 0; i < rqst[j].rq_npages; i++) {
struct bio_vec bvec;
bvec.bv_page = rqst[j].rq_pages[i];
rqst_page_get_length(&rqst[j], i, &bvec.bv_len,
&bvec.bv_offset);
iov_iter_bvec(&smb_msg.msg_iter, ITER_SOURCE,
&bvec, 1, bvec.bv_len);
if (iov_iter_count(&rqst[j].rq_iter) > 0) {
smb_msg.msg_iter = rqst[j].rq_iter;
rc = smb_send_kvec(server, &smb_msg, &sent);
if (rc < 0)
break;
total_len += sent;
}
}
}
unmask:
sigprocmask(SIG_SETMASK, &oldmask, NULL);
@ -1640,11 +1613,11 @@ int
cifs_discard_remaining_data(struct TCP_Server_Info *server)
{
unsigned int rfclen = server->pdu_size;
int remaining = rfclen + HEADER_PREAMBLE_SIZE(server) -
size_t remaining = rfclen + HEADER_PREAMBLE_SIZE(server) -
server->total_read;
while (remaining > 0) {
int length;
ssize_t length;
length = cifs_discard_from_socket(server,
min_t(size_t, remaining,
@ -1790,10 +1763,15 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid)
return cifs_readv_discard(server, mid);
}
length = rdata->read_into_pages(server, rdata, data_len);
if (length < 0)
return length;
#ifdef CONFIG_CIFS_SMB_DIRECT
if (rdata->mr)
length = data_len; /* An RDMA read is already done. */
else
#endif
length = cifs_read_iter_from_socket(server, &rdata->iter,
data_len);
if (length > 0)
rdata->got_bytes += length;
server->total_read += length;
cifs_dbg(FYI, "total_read=%u buflen=%u remaining=%u\n",