From 21b56c84775351ac66354c9b09fb429e5cdeceac Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 26 Apr 2021 20:50:05 -0400 Subject: [PATCH] iov_iter: get rid of separate bvec and xarray callbacks After the previous commit we have * xarray and bvec callbacks idential in all cases * both equivalent to kvec callback wrapped into kmap_local_page()/kunmap_local() pair. So we can pass only two (iovec and kvec) callbacks to iterate_and_advance() and let iterate_{bvec,xarray} wrap it into kmap_local_page()/kunmap_local_page(). Signed-off-by: Al Viro --- lib/iov_iter.c | 112 +++++++++++++------------------------------------ 1 file changed, 30 insertions(+), 82 deletions(-) diff --git a/lib/iov_iter.c b/lib/iov_iter.c index ac7682734df2..74e20d6d6e3d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -42,18 +42,20 @@ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ - __v.bv_offset = offset % PAGE_SIZE; \ - __v.bv_page = p->bv_page + offset / PAGE_SIZE; \ - __v.bv_len = min(min(n, p->bv_len - skip), \ + void *kaddr = kmap_local_page(p->bv_page + \ + offset / PAGE_SIZE); \ + __v.iov_base = kaddr + offset % PAGE_SIZE; \ + __v.iov_len = min(min(n, p->bv_len - skip), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ left = (STEP); \ - __v.bv_len -= left; \ - skip += __v.bv_len; \ + kunmap_local(kaddr); \ + __v.iov_len -= left; \ + skip += __v.iov_len; \ if (skip == p->bv_len) { \ skip = 0; \ p++; \ } \ - n -= __v.bv_len; \ + n -= __v.iov_len; \ if (left) \ break; \ } \ @@ -81,15 +83,16 @@ break; \ for (j = (head->index < index) ? index - head->index : 0; \ j < thp_nr_pages(head); j++) { \ - __v.bv_page = head + j; \ - offset = (i->xarray_start + skip) & ~PAGE_MASK; \ + void *kaddr = kmap_local_page(head + j); \ + offset = (i->xarray_start + skip) % PAGE_SIZE; \ + __v.iov_base = kaddr + offset; \ seg = PAGE_SIZE - offset; \ - __v.bv_offset = offset; \ - __v.bv_len = min(n, seg); \ + __v.iov_len = min(n, seg); \ left = (STEP); \ - __v.bv_len -= left; \ - n -= __v.bv_len; \ - skip += __v.bv_len; \ + kunmap_local(kaddr); \ + __v.iov_len -= left; \ + n -= __v.iov_len; \ + skip += __v.iov_len; \ if (left || n == 0) \ goto __out; \ } \ @@ -99,7 +102,7 @@ __out: \ n = wanted - n; \ } -#define __iterate_and_advance(i, n, v, I, B, K, X) { \ +#define __iterate_and_advance(i, n, v, I, K) { \ if (unlikely(i->count < n)) \ n = i->count; \ if (likely(n)) { \ @@ -112,8 +115,8 @@ __out: \ i->iov = iov; \ } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ - struct bio_vec v; \ - iterate_bvec(i, n, v, bvec, skip, (B)) \ + struct kvec v; \ + iterate_bvec(i, n, v, bvec, skip, (K)) \ i->nr_segs -= bvec - i->bvec; \ i->bvec = bvec; \ } else if (iov_iter_is_kvec(i)) { \ @@ -123,16 +126,15 @@ __out: \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ - struct bio_vec v; \ - iterate_xarray(i, n, v, skip, (X)) \ + struct kvec v; \ + iterate_xarray(i, n, v, skip, (K)) \ } \ i->count -= n; \ i->iov_offset = skip; \ } \ } -#define iterate_and_advance(i, n, v, I, B, K, X) \ - __iterate_and_advance(i, n, v, I, ((void)(B),0), \ - ((void)(K),0), ((void)(X),0)) +#define iterate_and_advance(i, n, v, I, K) \ + __iterate_and_advance(i, n, v, I, ((void)(K),0)) static int copyout(void __user *to, const void *from, size_t n) { @@ -612,11 +614,7 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) might_fault(); iterate_and_advance(i, bytes, v, copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), - memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len) + memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) ) return bytes; @@ -714,12 +712,8 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) __iterate_and_advance(i, bytes, v, copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), copy_mc_to_kernel(v.iov_base, (from += v.iov_len) - - v.iov_len, v.iov_len), - copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len) + - v.iov_len, v.iov_len) ) return bytes; @@ -738,11 +732,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) might_fault(); iterate_and_advance(i, bytes, v, copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) return bytes; @@ -759,11 +749,7 @@ size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) iterate_and_advance(i, bytes, v, __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) return bytes; @@ -795,12 +781,8 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) iterate_and_advance(i, bytes, v, __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, - v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + v.iov_len) ) return bytes; @@ -931,9 +913,7 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) return pipe_zero(bytes, i); iterate_and_advance(i, bytes, v, clear_user(v.iov_base, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len), - memset(v.iov_base, 0, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len) + memset(v.iov_base, 0, v.iov_len) ) return bytes; @@ -955,11 +935,7 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt } iterate_and_advance(i, bytes, v, copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) kunmap_atomic(kaddr); return bytes; @@ -1698,24 +1674,10 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, } next ? 0 : v.iov_len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len, sum, off); off += v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; }) ) *csum = sum; @@ -1750,24 +1712,10 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, } next ? 0 : v.iov_len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ sum = csum_and_memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len, sum, off); off += v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; }) ) csstate->csum = csum_shift(sum, csstate->off);