leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

block: allow bio_for_each_segment_all() to iterate over multi-page bvec

Browse Source 
This patch introduces one extra iterator variable to bio_for_each_segment_all(),
then we can allow bio_for_each_segment_all() to iterate over multi-page bvec.

Given it is just one mechannical & simple change on all bio_for_each_segment_all()
users, this patch does tree-wide change in one single patch, so that we can
avoid to use a temporary helper for this conversion.

Reviewed-by: Omar Sandoval <osandov@fb.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Ming Lei 2019-02-15 19:13:19 +08:00 committed by Jens Axboe
parent 2e1f4f4d24
commit 6dc4f100c1
27 changed files with 127 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
block/bio.c
View File

@ -1072,8 +1072,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
{ {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
ssize_t ret; ssize_t ret;
ret = copy_page_from_iter(bvec->bv_page, ret = copy_page_from_iter(bvec->bv_page,
@ -1103,8 +1104,9 @@ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
{ {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
ssize_t ret; ssize_t ret;
ret = copy_page_to_iter(bvec->bv_page, ret = copy_page_to_iter(bvec->bv_page,
@ -1126,8 +1128,9 @@ void bio_free_pages(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
__free_page(bvec->bv_page); __free_page(bvec->bv_page);
} }
EXPORT_SYMBOL(bio_free_pages); EXPORT_SYMBOL(bio_free_pages);
@ -1295,6 +1298,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
struct bio *bio; struct bio *bio;
int ret; int ret;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
if (!iov_iter_count(iter)) if (!iov_iter_count(iter))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
@ -1368,7 +1372,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
return bio; return bio;
out_unmap: out_unmap:
bio_for_each_segment_all(bvec, bio, j) { bio_for_each_segment_all(bvec, bio, j, iter_all) {
put_page(bvec->bv_page); put_page(bvec->bv_page);
} }
bio_put(bio); bio_put(bio);
@ -1379,11 +1383,12 @@ static void __bio_unmap_user(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
/* /*
* make sure we dirty pages we wrote to * make sure we dirty pages we wrote to
*/ */
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
if (bio_data_dir(bio) == READ) if (bio_data_dir(bio) == READ)
set_page_dirty_lock(bvec->bv_page); set_page_dirty_lock(bvec->bv_page);
@ -1475,8 +1480,9 @@ static void bio_copy_kern_endio_read(struct bio *bio)
char *p = bio->bi_private; char *p = bio->bi_private;
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
memcpy(p, page_address(bvec->bv_page), bvec->bv_len); memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
p += bvec->bv_len; p += bvec->bv_len;
} }
@ -1585,8 +1591,9 @@ void bio_set_pages_dirty(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
if (!PageCompound(bvec->bv_page)) if (!PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page); set_page_dirty_lock(bvec->bv_page);
} }
@ -1596,8 +1603,9 @@ static void bio_release_pages(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
put_page(bvec->bv_page); put_page(bvec->bv_page);
} }
@ -1644,8 +1652,9 @@ void bio_check_pages_dirty(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
unsigned long flags; unsigned long flags;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
goto defer; goto defer;
} }

6
block/bounce.c
View File

@ -165,11 +165,12 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool)
struct bio_vec *bvec, orig_vec; struct bio_vec *bvec, orig_vec;
int i; int i;
struct bvec_iter orig_iter = bio_orig->bi_iter; struct bvec_iter orig_iter = bio_orig->bi_iter;
struct bvec_iter_all iter_all;
/* /*
* free up bounce indirect pages used * free up bounce indirect pages used
*/ */
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
orig_vec = bio_iter_iovec(bio_orig, orig_iter); orig_vec = bio_iter_iovec(bio_orig, orig_iter);
if (bvec->bv_page != orig_vec.bv_page) { if (bvec->bv_page != orig_vec.bv_page) {
dec_zone_page_state(bvec->bv_page, NR_BOUNCE); dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
@ -294,6 +295,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
bool bounce = false; bool bounce = false;
int sectors = 0; int sectors = 0;
bool passthrough = bio_is_passthrough(*bio_orig); bool passthrough = bio_is_passthrough(*bio_orig);
struct bvec_iter_all iter_all;
bio_for_each_segment(from, *bio_orig, iter) { bio_for_each_segment(from, *bio_orig, iter) {
if (i++ < BIO_MAX_PAGES) if (i++ < BIO_MAX_PAGES)
@ -313,7 +315,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL : bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL :
&bounce_bio_set); &bounce_bio_set);
bio_for_each_segment_all(to, bio, i) { bio_for_each_segment_all(to, bio, i, iter_all) {
struct page *page = to->bv_page; struct page *page = to->bv_page;
if (page_to_pfn(page) <= q->limits.bounce_pfn) if (page_to_pfn(page) <= q->limits.bounce_pfn)

3
drivers/md/bcache/btree.c
View File

@ -432,8 +432,9 @@ static void do_btree_node_write(struct btree *b)
int j; int j;
struct bio_vec *bv; struct bio_vec *bv;
void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1)); void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1));
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, b->bio, j) bio_for_each_segment_all(bv, b->bio, j, iter_all)
memcpy(page_address(bv->bv_page), memcpy(page_address(bv->bv_page),
base + j * PAGE_SIZE, PAGE_SIZE); base + j * PAGE_SIZE, PAGE_SIZE);

3
drivers/md/dm-crypt.c
View File

@ -1447,8 +1447,9 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
{ {
unsigned int i; unsigned int i;
struct bio_vec *bv; struct bio_vec *bv;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, clone, i) { bio_for_each_segment_all(bv, clone, i, iter_all) {
BUG_ON(!bv->bv_page); BUG_ON(!bv->bv_page);
mempool_free(bv->bv_page, &cc->page_pool); mempool_free(bv->bv_page, &cc->page_pool);
} }

3
drivers/md/raid1.c
View File

@ -2112,13 +2112,14 @@ static void process_checks(struct r1bio *r1_bio)
struct page **spages = get_resync_pages(sbio)->pages; struct page **spages = get_resync_pages(sbio)->pages;
struct bio_vec *bi; struct bio_vec *bi;
int page_len[RESYNC_PAGES] = { 0 }; int page_len[RESYNC_PAGES] = { 0 };
struct bvec_iter_all iter_all;
if (sbio->bi_end_io != end_sync_read) if (sbio->bi_end_io != end_sync_read)
continue; continue;
/* Now we can 'fixup' the error value */ /* Now we can 'fixup' the error value */
sbio->bi_status = 0; sbio->bi_status = 0;
bio_for_each_segment_all(bi, sbio, j) bio_for_each_segment_all(bi, sbio, j, iter_all)
page_len[j] = bi->bv_len; page_len[j] = bi->bv_len;
if (!status) { if (!status) {

3
drivers/staging/erofs/data.c
View File

@ -20,8 +20,9 @@ static inline void read_endio(struct bio *bio)
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
const blk_status_t err = bio->bi_status; const blk_status_t err = bio->bi_status;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
/* page is already locked */ /* page is already locked */

3
drivers/staging/erofs/unzip_vle.c
View File

@ -830,8 +830,9 @@ static inline void z_erofs_vle_read_endio(struct bio *bio)
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
struct address_space *mc = NULL; struct address_space *mc = NULL;
#endif #endif
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
bool cachemngd = false; bool cachemngd = false;

6
fs/block_dev.c
View File

@ -211,6 +211,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
ssize_t ret; ssize_t ret;
blk_qc_t qc; blk_qc_t qc;
int i; int i;
struct bvec_iter_all iter_all;
if ((pos | iov_iter_alignment(iter)) & if ((pos | iov_iter_alignment(iter)) &
(bdev_logical_block_size(bdev) - 1)) (bdev_logical_block_size(bdev) - 1))
@ -260,7 +261,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
} }
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
bio_for_each_segment_all(bvec, &bio, i) { bio_for_each_segment_all(bvec, &bio, i, iter_all) {
if (should_dirty && !PageCompound(bvec->bv_page)) if (should_dirty && !PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page); set_page_dirty_lock(bvec->bv_page);
put_page(bvec->bv_page); put_page(bvec->bv_page);
@ -329,8 +330,9 @@ static void blkdev_bio_end_io(struct bio *bio)
} else { } else {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
put_page(bvec->bv_page); put_page(bvec->bv_page);
bio_put(bio); bio_put(bio);
} }

3
fs/btrfs/compression.c
View File

@ -162,13 +162,14 @@ csum_failed:
} else { } else {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
/* /*
* we have verified the checksum already, set page * we have verified the checksum already, set page
* checked so the end_io handlers know about it * checked so the end_io handlers know about it
*/ */
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, cb->orig_bio, i) bio_for_each_segment_all(bvec, cb->orig_bio, i, iter_all)
SetPageChecked(bvec->bv_page); SetPageChecked(bvec->bv_page);
bio_endio(cb->orig_bio); bio_endio(cb->orig_bio);

3
fs/btrfs/disk-io.c
View File

@ -832,9 +832,10 @@ static blk_status_t btree_csum_one_bio(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct btrfs_root *root; struct btrfs_root *root;
int i, ret = 0; int i, ret = 0;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
root = BTRFS_I(bvec->bv_page->mapping->host)->root; root = BTRFS_I(bvec->bv_page->mapping->host)->root;
ret = csum_dirty_buffer(root->fs_info, bvec->bv_page); ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
if (ret) if (ret)

9
fs/btrfs/extent_io.c
View File

@ -2422,9 +2422,10 @@ static void end_bio_extent_writepage(struct bio *bio)
u64 start; u64 start;
u64 end; u64 end;
int i; int i;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host; struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@ -2493,9 +2494,10 @@ static void end_bio_extent_readpage(struct bio *bio)
int mirror; int mirror;
int ret; int ret;
int i; int i;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host; struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@ -3635,9 +3637,10 @@ static void end_bio_extent_buffer_writepage(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct extent_buffer *eb; struct extent_buffer *eb;
int i, done; int i, done;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
eb = (struct extent_buffer *)page->private; eb = (struct extent_buffer *)page->private;

6
fs/btrfs/inode.c
View File

@ -7777,6 +7777,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct extent_io_tree *io_tree, *failure_tree; struct extent_io_tree *io_tree, *failure_tree;
int i; int i;
struct bvec_iter_all iter_all;
if (bio->bi_status) if (bio->bi_status)
goto end; goto end;
@ -7788,7 +7789,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
done->uptodate = 1; done->uptodate = 1;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
io_tree, done->start, bvec->bv_page, io_tree, done->start, bvec->bv_page,
btrfs_ino(BTRFS_I(inode)), 0); btrfs_ino(BTRFS_I(inode)), 0);
@ -7867,6 +7868,7 @@ static void btrfs_retry_endio(struct bio *bio)
int uptodate; int uptodate;
int ret; int ret;
int i; int i;
struct bvec_iter_all iter_all;
if (bio->bi_status) if (bio->bi_status)
goto end; goto end;
@ -7880,7 +7882,7 @@ static void btrfs_retry_endio(struct bio *bio)
failure_tree = &BTRFS_I(inode)->io_failure_tree; failure_tree = &BTRFS_I(inode)->io_failure_tree;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
bvec->bv_offset, done->start, bvec->bv_offset, done->start,
bvec->bv_len); bvec->bv_len);

3
fs/btrfs/raid56.c
View File

@ -1443,10 +1443,11 @@ static void set_bio_pages_uptodate(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
SetPageUptodate(bvec->bv_page); SetPageUptodate(bvec->bv_page);
} }

3
fs/crypto/bio.c
View File

@ -30,8 +30,9 @@ static void __fscrypt_decrypt_bio(struct bio *bio, bool done)
{ {
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
int ret = fscrypt_decrypt_page(page->mapping->host, page, int ret = fscrypt_decrypt_page(page->mapping->host, page,
PAGE_SIZE, 0, page->index); PAGE_SIZE, 0, page->index);

4
fs/direct-io.c
View File

@ -551,7 +551,9 @@ static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) { if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) {
bio_check_pages_dirty(bio); /* transfers ownership */ bio_check_pages_dirty(bio); /* transfers ownership */
} else { } else {
bio_for_each_segment_all(bvec, bio, i) { struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
if (dio->op == REQ_OP_READ && !PageCompound(page) && if (dio->op == REQ_OP_READ && !PageCompound(page) &&

3
fs/exofs/ore.c
View File

@ -420,8 +420,9 @@ static void _clear_bio(struct bio *bio)
{ {
struct bio_vec *bv; struct bio_vec *bv;
unsigned i; unsigned i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
unsigned this_count = bv->bv_len; unsigned this_count = bv->bv_len;
if (likely(PAGE_SIZE == this_count)) if (likely(PAGE_SIZE == this_count))

3
fs/exofs/ore_raid.c
View File

@ -468,11 +468,12 @@ static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
/* loop on all devices all pages */ /* loop on all devices all pages */
for (d = 0; d < ios->numdevs; d++) { for (d = 0; d < ios->numdevs; d++) {
struct bio *bio = ios->per_dev[d].bio; struct bio *bio = ios->per_dev[d].bio;
struct bvec_iter_all iter_all;
if (!bio) if (!bio)
continue; continue;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
SetPageUptodate(page); SetPageUptodate(page);

3
fs/ext4/page-io.c
View File

@ -63,8 +63,9 @@ static void ext4_finish_bio(struct bio *bio)
{ {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
#ifdef CONFIG_EXT4_FS_ENCRYPTION #ifdef CONFIG_EXT4_FS_ENCRYPTION
struct page *data_page = NULL; struct page *data_page = NULL;

3
fs/ext4/readpage.c
View File

@ -72,6 +72,7 @@ static void mpage_end_io(struct bio *bio)
{ {
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
if (ext4_bio_encrypted(bio)) { if (ext4_bio_encrypted(bio)) {
if (bio->bi_status) { if (bio->bi_status) {
@ -81,7 +82,7 @@ static void mpage_end_io(struct bio *bio)
return; return;
} }
} }
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
if (!bio->bi_status) { if (!bio->bi_status) {

9
fs/f2fs/data.c
View File

@ -87,8 +87,9 @@ static void __read_end_io(struct bio *bio)
struct page *page; struct page *page;
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
page = bv->bv_page; page = bv->bv_page;
/* PG_error was set if any post_read step failed */ /* PG_error was set if any post_read step failed */
@ -164,13 +165,14 @@ static void f2fs_write_end_io(struct bio *bio)
struct f2fs_sb_info *sbi = bio->bi_private; struct f2fs_sb_info *sbi = bio->bi_private;
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
if (time_to_inject(sbi, FAULT_WRITE_IO)) { if (time_to_inject(sbi, FAULT_WRITE_IO)) {
f2fs_show_injection_info(FAULT_WRITE_IO); f2fs_show_injection_info(FAULT_WRITE_IO);
bio->bi_status = BLK_STS_IOERR; bio->bi_status = BLK_STS_IOERR;
} }
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
enum count_type type = WB_DATA_TYPE(page); enum count_type type = WB_DATA_TYPE(page);
@ -347,6 +349,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
struct bio_vec *bvec; struct bio_vec *bvec;
struct page *target; struct page *target;
int i; int i;
struct bvec_iter_all iter_all;
if (!io->bio) if (!io->bio)
return false; return false;
@ -354,7 +357,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
if (!inode && !page && !ino) if (!inode && !page && !ino)
return true; return true;
bio_for_each_segment_all(bvec, io->bio, i) { bio_for_each_segment_all(bvec, io->bio, i, iter_all) {
if (bvec->bv_page->mapping) if (bvec->bv_page->mapping)
target = bvec->bv_page; target = bvec->bv_page;

9
fs/gfs2/lops.c
View File

@ -170,7 +170,8 @@ u64 gfs2_log_bmap(struct gfs2_sbd *sdp)
* that is pinned in the pagecache. * that is pinned in the pagecache.
*/ */
static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct bio_vec *bvec, static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,
struct bio_vec *bvec,
blk_status_t error) blk_status_t error)
{ {
struct buffer_head *bh, *next; struct buffer_head *bh, *next;
@ -208,6 +209,7 @@ static void gfs2_end_log_write(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct page *page; struct page *page;
int i; int i;
struct bvec_iter_all iter_all;
if (bio->bi_status) { if (bio->bi_status) {
fs_err(sdp, "Error %d writing to journal, jid=%u\n", fs_err(sdp, "Error %d writing to journal, jid=%u\n",
@ -215,7 +217,7 @@ static void gfs2_end_log_write(struct bio *bio)
wake_up(&sdp->sd_logd_waitq); wake_up(&sdp->sd_logd_waitq);
} }
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
page = bvec->bv_page; page = bvec->bv_page;
if (page_has_buffers(page)) if (page_has_buffers(page))
gfs2_end_log_write_bh(sdp, bvec, bio->bi_status); gfs2_end_log_write_bh(sdp, bvec, bio->bi_status);
@ -388,8 +390,9 @@ static void gfs2_end_log_read(struct bio *bio)
struct page *page; struct page *page;
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
page = bvec->bv_page; page = bvec->bv_page;
if (bio->bi_status) { if (bio->bi_status) {
int err = blk_status_to_errno(bio->bi_status); int err = blk_status_to_errno(bio->bi_status);

3
fs/gfs2/meta_io.c
View File

@ -190,8 +190,9 @@ static void gfs2_meta_read_endio(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
struct buffer_head *bh = page_buffers(page); struct buffer_head *bh = page_buffers(page);
unsigned int len = bvec->bv_len; unsigned int len = bvec->bv_len;

6
fs/iomap.c
View File

@ -267,8 +267,9 @@ iomap_read_end_io(struct bio *bio)
int error = blk_status_to_errno(bio->bi_status); int error = blk_status_to_errno(bio->bi_status);
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
iomap_read_page_end_io(bvec, error); iomap_read_page_end_io(bvec, error);
bio_put(bio); bio_put(bio);
} }
@ -1559,8 +1560,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)
} else { } else {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
put_page(bvec->bv_page); put_page(bvec->bv_page);
bio_put(bio); bio_put(bio);
} }

3
fs/mpage.c
View File

@ -48,8 +48,9 @@ static void mpage_end_io(struct bio *bio)
{ {
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
page_endio(page, bio_op(bio), page_endio(page, bio_op(bio),
blk_status_to_errno(bio->bi_status)); blk_status_to_errno(bio->bi_status));

5
fs/xfs/xfs_aops.c
View File

@ -62,7 +62,7 @@ xfs_find_daxdev_for_inode(
static void static void
xfs_finish_page_writeback( xfs_finish_page_writeback(
struct inode *inode, struct inode *inode,
struct bio_vec *bvec, struct bio_vec *bvec,
int error) int error)
{ {
struct iomap_page *iop = to_iomap_page(bvec->bv_page); struct iomap_page *iop = to_iomap_page(bvec->bv_page);
@ -98,6 +98,7 @@ xfs_destroy_ioend(
for (bio = &ioend->io_inline_bio; bio; bio = next) { for (bio = &ioend->io_inline_bio; bio; bio = next) {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
/* /*
* For the last bio, bi_private points to the ioend, so we * For the last bio, bi_private points to the ioend, so we
@ -109,7 +110,7 @@ xfs_destroy_ioend(
next = bio->bi_private; next = bio->bi_private;
/* walk each page on bio, ending page IO on them */ /* walk each page on bio, ending page IO on them */
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
xfs_finish_page_writeback(inode, bvec, error); xfs_finish_page_writeback(inode, bvec, error);
bio_put(bio); bio_put(bio);
} }

11
include/linux/bio.h
View File

@ -128,12 +128,19 @@ static inline bool bio_full(struct bio *bio)
return bio->bi_vcnt >= bio->bi_max_vecs; return bio->bi_vcnt >= bio->bi_max_vecs;
} }
#define mp_bvec_for_each_segment(bv, bvl, i, iter_all) \
for (bv = bvec_init_iter_all(&iter_all); \
(iter_all.done < (bvl)->bv_len) && \
(mp_bvec_next_segment((bvl), &iter_all), 1); \
iter_all.done += bv->bv_len, i += 1)
/* /*
* drivers should _never_ use the all version - the bio may have been split * drivers should _never_ use the all version - the bio may have been split
* before it got to the driver and the driver won't own all of it * before it got to the driver and the driver won't own all of it
*/ */
#define bio_for_each_segment_all(bvl, bio, i) \ #define bio_for_each_segment_all(bvl, bio, i, iter_all) \
for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++) for (i = 0, iter_all.idx = 0; iter_all.idx < (bio)->bi_vcnt; iter_all.idx++) \
mp_bvec_for_each_segment(bvl, &((bio)->bi_io_vec[iter_all.idx]), i, iter_all)
static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
unsigned bytes) unsigned bytes)

30
include/linux/bvec.h
View File

@ -45,6 +45,12 @@ struct bvec_iter {
current bvec */ current bvec */
}; };
struct bvec_iter_all {
struct bio_vec bv;
int idx;
unsigned done;
};
/* /*
* various member access, note that bio_data should of course not be used * various member access, note that bio_data should of course not be used
* on highmem page vectors * on highmem page vectors
@ -131,6 +137,30 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
.bi_bvec_done = 0, \ .bi_bvec_done = 0, \
} }
static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
{
iter_all->bv.bv_page = NULL;
iter_all->done = 0;
return &iter_all->bv;
}
static inline void mp_bvec_next_segment(const struct bio_vec *bvec,
struct bvec_iter_all *iter_all)
{
struct bio_vec *bv = &iter_all->bv;
if (bv->bv_page) {
bv->bv_page = nth_page(bv->bv_page, 1);
bv->bv_offset = 0;
} else {
bv->bv_page = bvec->bv_page;
bv->bv_offset = bvec->bv_offset;
}
bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
bvec->bv_len - iter_all->done);
}
/* /*
* Get the last single-page segment from the multi-page bvec and store it * Get the last single-page segment from the multi-page bvec and store it
* in @seg * in @seg