mirror of
https://github.com/torvalds/linux.git
synced 2024-11-29 15:41:36 +00:00
iov_iter: get rid of separate bvec and xarray callbacks
After the previous commit we have
* xarray and bvec callbacks idential in all cases
* both equivalent to kvec callback wrapped into
kmap_local_page()/kunmap_local() pair.
So we can pass only two (iovec and kvec) callbacks to
iterate_and_advance() and let iterate_{bvec,xarray} wrap
it into kmap_local_page()/kunmap_local_page().
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
This commit is contained in:
Al Viro
parent
1b4fb5ffd7
commit
21b56c8477
112
lib/iov_iter.c
112
lib/iov_iter.c
@ -42,18 +42,20 @@
|
||||
while (n) { \
|
||||
unsigned offset = p->bv_offset + skip; \
|
||||
unsigned left; \
|
||||
__v.bv_offset = offset % PAGE_SIZE; \
|
||||
__v.bv_page = p->bv_page + offset / PAGE_SIZE; \
|
||||
__v.bv_len = min(min(n, p->bv_len - skip), \
|
||||
void *kaddr = kmap_local_page(p->bv_page + \
|
||||
offset / PAGE_SIZE); \
|
||||
__v.iov_base = kaddr + offset % PAGE_SIZE; \
|
||||
__v.iov_len = min(min(n, p->bv_len - skip), \
|
||||
(size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \
|
||||
left = (STEP); \
|
||||
__v.bv_len -= left; \
|
||||
skip += __v.bv_len; \
|
||||
kunmap_local(kaddr); \
|
||||
__v.iov_len -= left; \
|
||||
skip += __v.iov_len; \
|
||||
if (skip == p->bv_len) { \
|
||||
skip = 0; \
|
||||
p++; \
|
||||
} \
|
||||
n -= __v.bv_len; \
|
||||
n -= __v.iov_len; \
|
||||
if (left) \
|
||||
break; \
|
||||
} \
|
||||
@ -81,15 +83,16 @@
|
||||
break; \
|
||||
for (j = (head->index < index) ? index - head->index : 0; \
|
||||
j < thp_nr_pages(head); j++) { \
|
||||
__v.bv_page = head + j; \
|
||||
offset = (i->xarray_start + skip) & ~PAGE_MASK; \
|
||||
void *kaddr = kmap_local_page(head + j); \
|
||||
offset = (i->xarray_start + skip) % PAGE_SIZE; \
|
||||
__v.iov_base = kaddr + offset; \
|
||||
seg = PAGE_SIZE - offset; \
|
||||
__v.bv_offset = offset; \
|
||||
__v.bv_len = min(n, seg); \
|
||||
__v.iov_len = min(n, seg); \
|
||||
left = (STEP); \
|
||||
__v.bv_len -= left; \
|
||||
n -= __v.bv_len; \
|
||||
skip += __v.bv_len; \
|
||||
kunmap_local(kaddr); \
|
||||
__v.iov_len -= left; \
|
||||
n -= __v.iov_len; \
|
||||
skip += __v.iov_len; \
|
||||
if (left || n == 0) \
|
||||
goto __out; \
|
||||
} \
|
||||
@ -99,7 +102,7 @@ __out: \
|
||||
n = wanted - n; \
|
||||
}
|
||||
|
||||
#define __iterate_and_advance(i, n, v, I, B, K, X) { \
|
||||
#define __iterate_and_advance(i, n, v, I, K) { \
|
||||
if (unlikely(i->count < n)) \
|
||||
n = i->count; \
|
||||
if (likely(n)) { \
|
||||
@ -112,8 +115,8 @@ __out: \
|
||||
i->iov = iov; \
|
||||
} else if (iov_iter_is_bvec(i)) { \
|
||||
const struct bio_vec *bvec = i->bvec; \
|
||||
struct bio_vec v; \
|
||||
iterate_bvec(i, n, v, bvec, skip, (B)) \
|
||||
struct kvec v; \
|
||||
iterate_bvec(i, n, v, bvec, skip, (K)) \
|
||||
i->nr_segs -= bvec - i->bvec; \
|
||||
i->bvec = bvec; \
|
||||
} else if (iov_iter_is_kvec(i)) { \
|
||||
@ -123,16 +126,15 @@ __out: \
|
||||
i->nr_segs -= kvec - i->kvec; \
|
||||
i->kvec = kvec; \
|
||||
} else if (iov_iter_is_xarray(i)) { \
|
||||
struct bio_vec v; \
|
||||
iterate_xarray(i, n, v, skip, (X)) \
|
||||
struct kvec v; \
|
||||
iterate_xarray(i, n, v, skip, (K)) \
|
||||
} \
|
||||
i->count -= n; \
|
||||
i->iov_offset = skip; \
|
||||
} \
|
||||
}
|
||||
#define iterate_and_advance(i, n, v, I, B, K, X) \
|
||||
__iterate_and_advance(i, n, v, I, ((void)(B),0), \
|
||||
((void)(K),0), ((void)(X),0))
|
||||
#define iterate_and_advance(i, n, v, I, K) \
|
||||
__iterate_and_advance(i, n, v, I, ((void)(K),0))
|
||||
|
||||
static int copyout(void __user *to, const void *from, size_t n)
|
||||
{
|
||||
@ -612,11 +614,7 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
|
||||
might_fault();
|
||||
iterate_and_advance(i, bytes, v,
|
||||
copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
|
||||
memcpy_to_page(v.bv_page, v.bv_offset,
|
||||
(from += v.bv_len) - v.bv_len, v.bv_len),
|
||||
memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
|
||||
memcpy_to_page(v.bv_page, v.bv_offset,
|
||||
(from += v.bv_len) - v.bv_len, v.bv_len)
|
||||
memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
|
||||
)
|
||||
|
||||
return bytes;
|
||||
@ -714,12 +712,8 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
|
||||
__iterate_and_advance(i, bytes, v,
|
||||
copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
|
||||
v.iov_len),
|
||||
copy_mc_to_page(v.bv_page, v.bv_offset,
|
||||
(from += v.bv_len) - v.bv_len, v.bv_len),
|
||||
copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
|
||||
- v.iov_len, v.iov_len),
|
||||
copy_mc_to_page(v.bv_page, v.bv_offset,
|
||||
(from += v.bv_len) - v.bv_len, v.bv_len)
|
||||
- v.iov_len, v.iov_len)
|
||||
)
|
||||
|
||||
return bytes;
|
||||
@ -738,11 +732,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
|
||||
might_fault();
|
||||
iterate_and_advance(i, bytes, v,
|
||||
copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
||||
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len),
|
||||
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
||||
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len)
|
||||
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
||||
)
|
||||
|
||||
return bytes;
|
||||
@ -759,11 +749,7 @@ size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
|
||||
iterate_and_advance(i, bytes, v,
|
||||
__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
|
||||
v.iov_base, v.iov_len),
|
||||
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len),
|
||||
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
||||
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len)
|
||||
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
||||
)
|
||||
|
||||
return bytes;
|
||||
@ -795,12 +781,8 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
|
||||
iterate_and_advance(i, bytes, v,
|
||||
__copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
|
||||
v.iov_base, v.iov_len),
|
||||
memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len),
|
||||
memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
|
||||
v.iov_len),
|
||||
memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len)
|
||||
v.iov_len)
|
||||
)
|
||||
|
||||
return bytes;
|
||||
@ -931,9 +913,7 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
|
||||
return pipe_zero(bytes, i);
|
||||
iterate_and_advance(i, bytes, v,
|
||||
clear_user(v.iov_base, v.iov_len),
|
||||
memzero_page(v.bv_page, v.bv_offset, v.bv_len),
|
||||
memset(v.iov_base, 0, v.iov_len),
|
||||
memzero_page(v.bv_page, v.bv_offset, v.bv_len)
|
||||
memset(v.iov_base, 0, v.iov_len)
|
||||
)
|
||||
|
||||
return bytes;
|
||||
@ -955,11 +935,7 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt
|
||||
}
|
||||
iterate_and_advance(i, bytes, v,
|
||||
copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
||||
memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len),
|
||||
memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
||||
memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
|
||||
v.bv_offset, v.bv_len)
|
||||
memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
||||
)
|
||||
kunmap_atomic(kaddr);
|
||||
return bytes;
|
||||
@ -1698,24 +1674,10 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
|
||||
}
|
||||
next ? 0 : v.iov_len;
|
||||
}), ({
|
||||
char *p = kmap_atomic(v.bv_page);
|
||||
sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
|
||||
p + v.bv_offset, v.bv_len,
|
||||
sum, off);
|
||||
kunmap_atomic(p);
|
||||
off += v.bv_len;
|
||||
}),({
|
||||
sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
|
||||
v.iov_base, v.iov_len,
|
||||
sum, off);
|
||||
off += v.iov_len;
|
||||
}), ({
|
||||
char *p = kmap_atomic(v.bv_page);
|
||||
sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
|
||||
p + v.bv_offset, v.bv_len,
|
||||
sum, off);
|
||||
kunmap_atomic(p);
|
||||
off += v.bv_len;
|
||||
})
|
||||
)
|
||||
*csum = sum;
|
||||
@ -1750,24 +1712,10 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
|
||||
}
|
||||
next ? 0 : v.iov_len;
|
||||
}), ({
|
||||
char *p = kmap_atomic(v.bv_page);
|
||||
sum = csum_and_memcpy(p + v.bv_offset,
|
||||
(from += v.bv_len) - v.bv_len,
|
||||
v.bv_len, sum, off);
|
||||
kunmap_atomic(p);
|
||||
off += v.bv_len;
|
||||
}),({
|
||||
sum = csum_and_memcpy(v.iov_base,
|
||||
(from += v.iov_len) - v.iov_len,
|
||||
v.iov_len, sum, off);
|
||||
off += v.iov_len;
|
||||
}), ({
|
||||
char *p = kmap_atomic(v.bv_page);
|
||||
sum = csum_and_memcpy(p + v.bv_offset,
|
||||
(from += v.bv_len) - v.bv_len,
|
||||
v.bv_len, sum, off);
|
||||
kunmap_atomic(p);
|
||||
off += v.bv_len;
|
||||
})
|
||||
)
|
||||
csstate->csum = csum_shift(sum, csstate->off);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user