mirror of
https://github.com/torvalds/linux.git
synced 2024-11-23 04:31:50 +00:00
7554a8bb6d
This adds a new helper, bch2_folio_reservation_get_partial(), which reserves as many blocks as possible and may return partial success. __bch2_buffered_write() is switched to the new helper - this fixes fstests generic/275, the write until -ENOSPC test. generic/230 now fails: this appears to be a test bug, where xfs_io isn't looping after a partial write to get the error code. Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
1158 lines
28 KiB
C
1158 lines
28 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#ifndef NO_BCACHEFS_FS
|
|
|
|
#include "bcachefs.h"
|
|
#include "alloc_foreground.h"
|
|
#include "bkey_buf.h"
|
|
#include "fs-io.h"
|
|
#include "fs-io-buffered.h"
|
|
#include "fs-io-direct.h"
|
|
#include "fs-io-pagecache.h"
|
|
#include "io_read.h"
|
|
#include "io_write.h"
|
|
|
|
#include <linux/backing-dev.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/writeback.h>
|
|
|
|
static inline bool bio_full(struct bio *bio, unsigned len)
|
|
{
|
|
if (bio->bi_vcnt >= bio->bi_max_vecs)
|
|
return true;
|
|
if (bio->bi_iter.bi_size > UINT_MAX - len)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
/* readpage(s): */
|
|
|
|
static void bch2_readpages_end_io(struct bio *bio)
|
|
{
|
|
struct folio_iter fi;
|
|
|
|
bio_for_each_folio_all(fi, bio)
|
|
folio_end_read(fi.folio, bio->bi_status == BLK_STS_OK);
|
|
|
|
bio_put(bio);
|
|
}
|
|
|
|
struct readpages_iter {
|
|
struct address_space *mapping;
|
|
unsigned idx;
|
|
folios folios;
|
|
};
|
|
|
|
static int readpages_iter_init(struct readpages_iter *iter,
|
|
struct readahead_control *ractl)
|
|
{
|
|
struct folio *folio;
|
|
|
|
*iter = (struct readpages_iter) { ractl->mapping };
|
|
|
|
while ((folio = __readahead_folio(ractl))) {
|
|
if (!bch2_folio_create(folio, GFP_KERNEL) ||
|
|
darray_push(&iter->folios, folio)) {
|
|
bch2_folio_release(folio);
|
|
ractl->_nr_pages += folio_nr_pages(folio);
|
|
ractl->_index -= folio_nr_pages(folio);
|
|
return iter->folios.nr ? 0 : -ENOMEM;
|
|
}
|
|
|
|
folio_put(folio);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
|
|
{
|
|
if (iter->idx >= iter->folios.nr)
|
|
return NULL;
|
|
return iter->folios.data[iter->idx];
|
|
}
|
|
|
|
static inline void readpage_iter_advance(struct readpages_iter *iter)
|
|
{
|
|
iter->idx++;
|
|
}
|
|
|
|
static bool extent_partial_reads_expensive(struct bkey_s_c k)
|
|
{
|
|
struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
|
|
struct bch_extent_crc_unpacked crc;
|
|
const union bch_extent_entry *i;
|
|
|
|
bkey_for_each_crc(k.k, ptrs, crc, i)
|
|
if (crc.csum_type || crc.compression_type)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
static int readpage_bio_extend(struct btree_trans *trans,
|
|
struct readpages_iter *iter,
|
|
struct bio *bio,
|
|
unsigned sectors_this_extent,
|
|
bool get_more)
|
|
{
|
|
/* Don't hold btree locks while allocating memory: */
|
|
bch2_trans_unlock(trans);
|
|
|
|
while (bio_sectors(bio) < sectors_this_extent &&
|
|
bio->bi_vcnt < bio->bi_max_vecs) {
|
|
struct folio *folio = readpage_iter_peek(iter);
|
|
int ret;
|
|
|
|
if (folio) {
|
|
readpage_iter_advance(iter);
|
|
} else {
|
|
pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
|
|
|
|
if (!get_more)
|
|
break;
|
|
|
|
folio = xa_load(&iter->mapping->i_pages, folio_offset);
|
|
if (folio && !xa_is_value(folio))
|
|
break;
|
|
|
|
folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
|
|
if (!folio)
|
|
break;
|
|
|
|
if (!__bch2_folio_create(folio, GFP_KERNEL)) {
|
|
folio_put(folio);
|
|
break;
|
|
}
|
|
|
|
ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
|
|
if (ret) {
|
|
__bch2_folio_release(folio);
|
|
folio_put(folio);
|
|
break;
|
|
}
|
|
|
|
folio_put(folio);
|
|
}
|
|
|
|
BUG_ON(folio_sector(folio) != bio_end_sector(bio));
|
|
|
|
BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
|
|
}
|
|
|
|
return bch2_trans_relock(trans);
|
|
}
|
|
|
|
static void bchfs_read(struct btree_trans *trans,
|
|
struct bch_read_bio *rbio,
|
|
subvol_inum inum,
|
|
struct readpages_iter *readpages_iter)
|
|
{
|
|
struct bch_fs *c = trans->c;
|
|
struct btree_iter iter;
|
|
struct bkey_buf sk;
|
|
int flags = BCH_READ_RETRY_IF_STALE|
|
|
BCH_READ_MAY_PROMOTE;
|
|
u32 snapshot;
|
|
int ret = 0;
|
|
|
|
rbio->c = c;
|
|
rbio->start_time = local_clock();
|
|
rbio->subvol = inum.subvol;
|
|
|
|
bch2_bkey_buf_init(&sk);
|
|
retry:
|
|
bch2_trans_begin(trans);
|
|
iter = (struct btree_iter) { NULL };
|
|
|
|
ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
|
|
if (ret)
|
|
goto err;
|
|
|
|
bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
|
|
SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
|
|
BTREE_ITER_slots);
|
|
while (1) {
|
|
struct bkey_s_c k;
|
|
unsigned bytes, sectors, offset_into_extent;
|
|
enum btree_id data_btree = BTREE_ID_extents;
|
|
|
|
/*
|
|
* read_extent -> io_time_reset may cause a transaction restart
|
|
* without returning an error, we need to check for that here:
|
|
*/
|
|
ret = bch2_trans_relock(trans);
|
|
if (ret)
|
|
break;
|
|
|
|
bch2_btree_iter_set_pos(&iter,
|
|
POS(inum.inum, rbio->bio.bi_iter.bi_sector));
|
|
|
|
k = bch2_btree_iter_peek_slot(&iter);
|
|
ret = bkey_err(k);
|
|
if (ret)
|
|
break;
|
|
|
|
offset_into_extent = iter.pos.offset -
|
|
bkey_start_offset(k.k);
|
|
sectors = k.k->size - offset_into_extent;
|
|
|
|
bch2_bkey_buf_reassemble(&sk, c, k);
|
|
|
|
ret = bch2_read_indirect_extent(trans, &data_btree,
|
|
&offset_into_extent, &sk);
|
|
if (ret)
|
|
break;
|
|
|
|
k = bkey_i_to_s_c(sk.k);
|
|
|
|
sectors = min(sectors, k.k->size - offset_into_extent);
|
|
|
|
if (readpages_iter) {
|
|
ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
|
|
extent_partial_reads_expensive(k));
|
|
if (ret)
|
|
break;
|
|
}
|
|
|
|
bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
|
|
swap(rbio->bio.bi_iter.bi_size, bytes);
|
|
|
|
if (rbio->bio.bi_iter.bi_size == bytes)
|
|
flags |= BCH_READ_LAST_FRAGMENT;
|
|
|
|
bch2_bio_page_state_set(&rbio->bio, k);
|
|
|
|
bch2_read_extent(trans, rbio, iter.pos,
|
|
data_btree, k, offset_into_extent, flags);
|
|
|
|
if (flags & BCH_READ_LAST_FRAGMENT)
|
|
break;
|
|
|
|
swap(rbio->bio.bi_iter.bi_size, bytes);
|
|
bio_advance(&rbio->bio, bytes);
|
|
|
|
ret = btree_trans_too_many_iters(trans);
|
|
if (ret)
|
|
break;
|
|
}
|
|
err:
|
|
bch2_trans_iter_exit(trans, &iter);
|
|
|
|
if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
|
|
goto retry;
|
|
|
|
if (ret) {
|
|
bch_err_inum_offset_ratelimited(c,
|
|
iter.pos.inode,
|
|
iter.pos.offset << 9,
|
|
"read error %i from btree lookup", ret);
|
|
rbio->bio.bi_status = BLK_STS_IOERR;
|
|
bio_endio(&rbio->bio);
|
|
}
|
|
|
|
bch2_bkey_buf_exit(&sk, c);
|
|
}
|
|
|
|
void bch2_readahead(struct readahead_control *ractl)
|
|
{
|
|
struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
|
|
struct bch_fs *c = inode->v.i_sb->s_fs_info;
|
|
struct bch_io_opts opts;
|
|
struct folio *folio;
|
|
struct readpages_iter readpages_iter;
|
|
|
|
bch2_inode_opts_get(&opts, c, &inode->ei_inode);
|
|
|
|
int ret = readpages_iter_init(&readpages_iter, ractl);
|
|
if (ret)
|
|
return;
|
|
|
|
bch2_pagecache_add_get(inode);
|
|
|
|
struct btree_trans *trans = bch2_trans_get(c);
|
|
while ((folio = readpage_iter_peek(&readpages_iter))) {
|
|
unsigned n = min_t(unsigned,
|
|
readpages_iter.folios.nr -
|
|
readpages_iter.idx,
|
|
BIO_MAX_VECS);
|
|
struct bch_read_bio *rbio =
|
|
rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
|
|
GFP_KERNEL, &c->bio_read),
|
|
opts);
|
|
|
|
readpage_iter_advance(&readpages_iter);
|
|
|
|
rbio->bio.bi_iter.bi_sector = folio_sector(folio);
|
|
rbio->bio.bi_end_io = bch2_readpages_end_io;
|
|
BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
|
|
|
|
bchfs_read(trans, rbio, inode_inum(inode),
|
|
&readpages_iter);
|
|
bch2_trans_unlock(trans);
|
|
}
|
|
bch2_trans_put(trans);
|
|
|
|
bch2_pagecache_add_put(inode);
|
|
|
|
darray_exit(&readpages_iter.folios);
|
|
}
|
|
|
|
static void bch2_read_single_folio_end_io(struct bio *bio)
|
|
{
|
|
complete(bio->bi_private);
|
|
}
|
|
|
|
int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)
|
|
{
|
|
struct bch_inode_info *inode = to_bch_ei(mapping->host);
|
|
struct bch_fs *c = inode->v.i_sb->s_fs_info;
|
|
struct bch_read_bio *rbio;
|
|
struct bch_io_opts opts;
|
|
int ret;
|
|
DECLARE_COMPLETION_ONSTACK(done);
|
|
|
|
if (!bch2_folio_create(folio, GFP_KERNEL))
|
|
return -ENOMEM;
|
|
|
|
bch2_inode_opts_get(&opts, c, &inode->ei_inode);
|
|
|
|
rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
|
|
opts);
|
|
rbio->bio.bi_private = &done;
|
|
rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
|
|
|
|
rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
|
|
rbio->bio.bi_iter.bi_sector = folio_sector(folio);
|
|
BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
|
|
|
|
bch2_trans_run(c, (bchfs_read(trans, rbio, inode_inum(inode), NULL), 0));
|
|
wait_for_completion(&done);
|
|
|
|
ret = blk_status_to_errno(rbio->bio.bi_status);
|
|
bio_put(&rbio->bio);
|
|
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
folio_mark_uptodate(folio);
|
|
return 0;
|
|
}
|
|
|
|
int bch2_read_folio(struct file *file, struct folio *folio)
|
|
{
|
|
int ret;
|
|
|
|
ret = bch2_read_single_folio(folio, folio->mapping);
|
|
folio_unlock(folio);
|
|
return bch2_err_class(ret);
|
|
}
|
|
|
|
/* writepages: */
|
|
|
|
struct bch_writepage_io {
|
|
struct bch_inode_info *inode;
|
|
|
|
/* must be last: */
|
|
struct bch_write_op op;
|
|
};
|
|
|
|
struct bch_writepage_state {
|
|
struct bch_writepage_io *io;
|
|
struct bch_io_opts opts;
|
|
struct bch_folio_sector *tmp;
|
|
unsigned tmp_sectors;
|
|
};
|
|
|
|
static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
|
|
struct bch_inode_info *inode)
|
|
{
|
|
struct bch_writepage_state ret = { 0 };
|
|
|
|
bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Determine when a writepage io is full. We have to limit writepage bios to a
|
|
* single page per bvec (i.e. 1MB with 4k pages) because that is the limit to
|
|
* what the bounce path in bch2_write_extent() can handle. In theory we could
|
|
* loosen this restriction for non-bounce I/O, but we don't have that context
|
|
* here. Ideally, we can up this limit and make it configurable in the future
|
|
* when the bounce path can be enhanced to accommodate larger source bios.
|
|
*/
|
|
static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len)
|
|
{
|
|
struct bio *bio = &io->op.wbio.bio;
|
|
return bio_full(bio, len) ||
|
|
(bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE);
|
|
}
|
|
|
|
static void bch2_writepage_io_done(struct bch_write_op *op)
|
|
{
|
|
struct bch_writepage_io *io =
|
|
container_of(op, struct bch_writepage_io, op);
|
|
struct bch_fs *c = io->op.c;
|
|
struct bio *bio = &io->op.wbio.bio;
|
|
struct folio_iter fi;
|
|
unsigned i;
|
|
|
|
if (io->op.error) {
|
|
set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
|
|
|
|
bio_for_each_folio_all(fi, bio) {
|
|
struct bch_folio *s;
|
|
|
|
mapping_set_error(fi.folio->mapping, -EIO);
|
|
|
|
s = __bch2_folio(fi.folio);
|
|
spin_lock(&s->lock);
|
|
for (i = 0; i < folio_sectors(fi.folio); i++)
|
|
s->s[i].nr_replicas = 0;
|
|
spin_unlock(&s->lock);
|
|
}
|
|
}
|
|
|
|
if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
|
|
bio_for_each_folio_all(fi, bio) {
|
|
struct bch_folio *s;
|
|
|
|
s = __bch2_folio(fi.folio);
|
|
spin_lock(&s->lock);
|
|
for (i = 0; i < folio_sectors(fi.folio); i++)
|
|
s->s[i].nr_replicas = 0;
|
|
spin_unlock(&s->lock);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* racing with fallocate can cause us to add fewer sectors than
|
|
* expected - but we shouldn't add more sectors than expected:
|
|
*/
|
|
WARN_ON_ONCE(io->op.i_sectors_delta > 0);
|
|
|
|
/*
|
|
* (error (due to going RO) halfway through a page can screw that up
|
|
* slightly)
|
|
* XXX wtf?
|
|
BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
|
|
*/
|
|
|
|
/*
|
|
* The writeback flag is effectively our ref on the inode -
|
|
* fixup i_blocks before calling folio_end_writeback:
|
|
*/
|
|
bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
|
|
|
|
bio_for_each_folio_all(fi, bio) {
|
|
struct bch_folio *s = __bch2_folio(fi.folio);
|
|
|
|
if (atomic_dec_and_test(&s->write_count))
|
|
folio_end_writeback(fi.folio);
|
|
}
|
|
|
|
bio_put(&io->op.wbio.bio);
|
|
}
|
|
|
|
static void bch2_writepage_do_io(struct bch_writepage_state *w)
|
|
{
|
|
struct bch_writepage_io *io = w->io;
|
|
|
|
w->io = NULL;
|
|
closure_call(&io->op.cl, bch2_write, NULL, NULL);
|
|
}
|
|
|
|
/*
|
|
* Get a bch_writepage_io and add @page to it - appending to an existing one if
|
|
* possible, else allocating a new one:
|
|
*/
|
|
static void bch2_writepage_io_alloc(struct bch_fs *c,
|
|
struct writeback_control *wbc,
|
|
struct bch_writepage_state *w,
|
|
struct bch_inode_info *inode,
|
|
u64 sector,
|
|
unsigned nr_replicas)
|
|
{
|
|
struct bch_write_op *op;
|
|
|
|
w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
|
|
REQ_OP_WRITE,
|
|
GFP_KERNEL,
|
|
&c->writepage_bioset),
|
|
struct bch_writepage_io, op.wbio.bio);
|
|
|
|
w->io->inode = inode;
|
|
op = &w->io->op;
|
|
bch2_write_op_init(op, c, w->opts);
|
|
op->target = w->opts.foreground_target;
|
|
op->nr_replicas = nr_replicas;
|
|
op->res.nr_replicas = nr_replicas;
|
|
op->write_point = writepoint_hashed(inode->ei_last_dirtied);
|
|
op->subvol = inode->ei_subvol;
|
|
op->pos = POS(inode->v.i_ino, sector);
|
|
op->end_io = bch2_writepage_io_done;
|
|
op->devs_need_flush = &inode->ei_devs_need_flush;
|
|
op->wbio.bio.bi_iter.bi_sector = sector;
|
|
op->wbio.bio.bi_opf = wbc_to_write_flags(wbc);
|
|
}
|
|
|
|
static int __bch2_writepage(struct folio *folio,
|
|
struct writeback_control *wbc,
|
|
void *data)
|
|
{
|
|
struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
|
|
struct bch_fs *c = inode->v.i_sb->s_fs_info;
|
|
struct bch_writepage_state *w = data;
|
|
struct bch_folio *s;
|
|
unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
|
|
loff_t i_size = i_size_read(&inode->v);
|
|
int ret;
|
|
|
|
EBUG_ON(!folio_test_uptodate(folio));
|
|
|
|
/* Is the folio fully inside i_size? */
|
|
if (folio_end_pos(folio) <= i_size)
|
|
goto do_io;
|
|
|
|
/* Is the folio fully outside i_size? (truncate in progress) */
|
|
if (folio_pos(folio) >= i_size) {
|
|
folio_unlock(folio);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* The folio straddles i_size. It must be zeroed out on each and every
|
|
* writepage invocation because it may be mmapped. "A file is mapped
|
|
* in multiples of the folio size. For a file that is not a multiple of
|
|
* the folio size, the remaining memory is zeroed when mapped, and
|
|
* writes to that region are not written out to the file."
|
|
*/
|
|
folio_zero_segment(folio,
|
|
i_size - folio_pos(folio),
|
|
folio_size(folio));
|
|
do_io:
|
|
f_sectors = folio_sectors(folio);
|
|
s = bch2_folio(folio);
|
|
|
|
if (f_sectors > w->tmp_sectors) {
|
|
kfree(w->tmp);
|
|
w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL);
|
|
w->tmp_sectors = f_sectors;
|
|
}
|
|
|
|
/*
|
|
* Things get really hairy with errors during writeback:
|
|
*/
|
|
ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
|
|
BUG_ON(ret);
|
|
|
|
/* Before unlocking the page, get copy of reservations: */
|
|
spin_lock(&s->lock);
|
|
memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
|
|
|
|
for (i = 0; i < f_sectors; i++) {
|
|
if (s->s[i].state < SECTOR_dirty)
|
|
continue;
|
|
|
|
nr_replicas_this_write =
|
|
min_t(unsigned, nr_replicas_this_write,
|
|
s->s[i].nr_replicas +
|
|
s->s[i].replicas_reserved);
|
|
}
|
|
|
|
for (i = 0; i < f_sectors; i++) {
|
|
if (s->s[i].state < SECTOR_dirty)
|
|
continue;
|
|
|
|
s->s[i].nr_replicas = w->opts.compression
|
|
? 0 : nr_replicas_this_write;
|
|
|
|
s->s[i].replicas_reserved = 0;
|
|
bch2_folio_sector_set(folio, s, i, SECTOR_allocated);
|
|
}
|
|
spin_unlock(&s->lock);
|
|
|
|
BUG_ON(atomic_read(&s->write_count));
|
|
atomic_set(&s->write_count, 1);
|
|
|
|
BUG_ON(folio_test_writeback(folio));
|
|
folio_start_writeback(folio);
|
|
|
|
folio_unlock(folio);
|
|
|
|
offset = 0;
|
|
while (1) {
|
|
unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
|
|
u64 sector;
|
|
|
|
while (offset < f_sectors &&
|
|
w->tmp[offset].state < SECTOR_dirty)
|
|
offset++;
|
|
|
|
if (offset == f_sectors)
|
|
break;
|
|
|
|
while (offset + sectors < f_sectors &&
|
|
w->tmp[offset + sectors].state >= SECTOR_dirty) {
|
|
reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
|
|
dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
|
|
sectors++;
|
|
}
|
|
BUG_ON(!sectors);
|
|
|
|
sector = folio_sector(folio) + offset;
|
|
|
|
if (w->io &&
|
|
(w->io->op.res.nr_replicas != nr_replicas_this_write ||
|
|
bch_io_full(w->io, sectors << 9) ||
|
|
bio_end_sector(&w->io->op.wbio.bio) != sector))
|
|
bch2_writepage_do_io(w);
|
|
|
|
if (!w->io)
|
|
bch2_writepage_io_alloc(c, wbc, w, inode, sector,
|
|
nr_replicas_this_write);
|
|
|
|
atomic_inc(&s->write_count);
|
|
|
|
BUG_ON(inode != w->io->inode);
|
|
BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
|
|
sectors << 9, offset << 9));
|
|
|
|
/* Check for writing past i_size: */
|
|
WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
|
|
round_up(i_size, block_bytes(c)) &&
|
|
!test_bit(BCH_FS_emergency_ro, &c->flags),
|
|
"writing past i_size: %llu > %llu (unrounded %llu)\n",
|
|
bio_end_sector(&w->io->op.wbio.bio) << 9,
|
|
round_up(i_size, block_bytes(c)),
|
|
i_size);
|
|
|
|
w->io->op.res.sectors += reserved_sectors;
|
|
w->io->op.i_sectors_delta -= dirty_sectors;
|
|
w->io->op.new_i_size = i_size;
|
|
|
|
offset += sectors;
|
|
}
|
|
|
|
if (atomic_dec_and_test(&s->write_count))
|
|
folio_end_writeback(folio);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
|
|
{
|
|
struct bch_fs *c = mapping->host->i_sb->s_fs_info;
|
|
struct bch_writepage_state w =
|
|
bch_writepage_state_init(c, to_bch_ei(mapping->host));
|
|
struct blk_plug plug;
|
|
int ret;
|
|
|
|
blk_start_plug(&plug);
|
|
ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
|
|
if (w.io)
|
|
bch2_writepage_do_io(&w);
|
|
blk_finish_plug(&plug);
|
|
kfree(w.tmp);
|
|
return bch2_err_class(ret);
|
|
}
|
|
|
|
/* buffered writes: */
|
|
|
|
int bch2_write_begin(struct file *file, struct address_space *mapping,
|
|
loff_t pos, unsigned len,
|
|
struct page **pagep, void **fsdata)
|
|
{
|
|
struct bch_inode_info *inode = to_bch_ei(mapping->host);
|
|
struct bch_fs *c = inode->v.i_sb->s_fs_info;
|
|
struct bch2_folio_reservation *res;
|
|
struct folio *folio;
|
|
unsigned offset;
|
|
int ret = -ENOMEM;
|
|
|
|
res = kmalloc(sizeof(*res), GFP_KERNEL);
|
|
if (!res)
|
|
return -ENOMEM;
|
|
|
|
bch2_folio_reservation_init(c, inode, res);
|
|
*fsdata = res;
|
|
|
|
bch2_pagecache_add_get(inode);
|
|
|
|
folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
|
|
FGP_WRITEBEGIN | fgf_set_order(len),
|
|
mapping_gfp_mask(mapping));
|
|
if (IS_ERR_OR_NULL(folio))
|
|
goto err_unlock;
|
|
|
|
offset = pos - folio_pos(folio);
|
|
len = min_t(size_t, len, folio_end_pos(folio) - pos);
|
|
|
|
if (folio_test_uptodate(folio))
|
|
goto out;
|
|
|
|
/* If we're writing entire folio, don't need to read it in first: */
|
|
if (!offset && len == folio_size(folio))
|
|
goto out;
|
|
|
|
if (!offset && pos + len >= inode->v.i_size) {
|
|
folio_zero_segment(folio, len, folio_size(folio));
|
|
flush_dcache_folio(folio);
|
|
goto out;
|
|
}
|
|
|
|
if (folio_pos(folio) >= inode->v.i_size) {
|
|
folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
|
|
flush_dcache_folio(folio);
|
|
goto out;
|
|
}
|
|
readpage:
|
|
ret = bch2_read_single_folio(folio, mapping);
|
|
if (ret)
|
|
goto err;
|
|
out:
|
|
ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
|
|
if (ret)
|
|
goto err;
|
|
|
|
ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
|
|
if (ret) {
|
|
if (!folio_test_uptodate(folio)) {
|
|
/*
|
|
* If the folio hasn't been read in, we won't know if we
|
|
* actually need a reservation - we don't actually need
|
|
* to read here, we just need to check if the folio is
|
|
* fully backed by uncompressed data:
|
|
*/
|
|
goto readpage;
|
|
}
|
|
|
|
goto err;
|
|
}
|
|
|
|
*pagep = &folio->page;
|
|
return 0;
|
|
err:
|
|
folio_unlock(folio);
|
|
folio_put(folio);
|
|
*pagep = NULL;
|
|
err_unlock:
|
|
bch2_pagecache_add_put(inode);
|
|
kfree(res);
|
|
*fsdata = NULL;
|
|
return bch2_err_class(ret);
|
|
}
|
|
|
|
int bch2_write_end(struct file *file, struct address_space *mapping,
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
struct page *page, void *fsdata)
|
|
{
|
|
struct bch_inode_info *inode = to_bch_ei(mapping->host);
|
|
struct bch_fs *c = inode->v.i_sb->s_fs_info;
|
|
struct bch2_folio_reservation *res = fsdata;
|
|
struct folio *folio = page_folio(page);
|
|
unsigned offset = pos - folio_pos(folio);
|
|
|
|
lockdep_assert_held(&inode->v.i_rwsem);
|
|
BUG_ON(offset + copied > folio_size(folio));
|
|
|
|
if (unlikely(copied < len && !folio_test_uptodate(folio))) {
|
|
/*
|
|
* The folio needs to be read in, but that would destroy
|
|
* our partial write - simplest thing is to just force
|
|
* userspace to redo the write:
|
|
*/
|
|
folio_zero_range(folio, 0, folio_size(folio));
|
|
flush_dcache_folio(folio);
|
|
copied = 0;
|
|
}
|
|
|
|
spin_lock(&inode->v.i_lock);
|
|
if (pos + copied > inode->v.i_size)
|
|
i_size_write(&inode->v, pos + copied);
|
|
spin_unlock(&inode->v.i_lock);
|
|
|
|
if (copied) {
|
|
if (!folio_test_uptodate(folio))
|
|
folio_mark_uptodate(folio);
|
|
|
|
bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
|
|
|
|
inode->ei_last_dirtied = (unsigned long) current;
|
|
}
|
|
|
|
folio_unlock(folio);
|
|
folio_put(folio);
|
|
bch2_pagecache_add_put(inode);
|
|
|
|
bch2_folio_reservation_put(c, inode, res);
|
|
kfree(res);
|
|
|
|
return copied;
|
|
}
|
|
|
|
static noinline void folios_trunc(folios *fs, struct folio **fi)
|
|
{
|
|
while (fs->data + fs->nr > fi) {
|
|
struct folio *f = darray_pop(fs);
|
|
|
|
folio_unlock(f);
|
|
folio_put(f);
|
|
}
|
|
}
|
|
|
|
static int __bch2_buffered_write(struct bch_inode_info *inode,
|
|
struct address_space *mapping,
|
|
struct iov_iter *iter,
|
|
loff_t pos, unsigned len,
|
|
bool inode_locked)
|
|
{
|
|
struct bch_fs *c = inode->v.i_sb->s_fs_info;
|
|
struct bch2_folio_reservation res;
|
|
folios fs;
|
|
struct folio *f;
|
|
unsigned copied = 0, f_offset, f_copied;
|
|
u64 end = pos + len, f_pos, f_len;
|
|
loff_t last_folio_pos = inode->v.i_size;
|
|
int ret = 0;
|
|
|
|
BUG_ON(!len);
|
|
|
|
bch2_folio_reservation_init(c, inode, &res);
|
|
darray_init(&fs);
|
|
|
|
ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,
|
|
FGP_WRITEBEGIN | fgf_set_order(len),
|
|
mapping_gfp_mask(mapping), &fs);
|
|
if (ret)
|
|
goto out;
|
|
|
|
BUG_ON(!fs.nr);
|
|
|
|
/*
|
|
* If we're not using the inode lock, we need to lock all the folios for
|
|
* atomiticity of writes vs. other writes:
|
|
*/
|
|
if (!inode_locked && folio_end_pos(darray_last(fs)) < end) {
|
|
ret = -BCH_ERR_need_inode_lock;
|
|
goto out;
|
|
}
|
|
|
|
f = darray_first(fs);
|
|
if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
|
|
ret = bch2_read_single_folio(f, mapping);
|
|
if (ret)
|
|
goto out;
|
|
}
|
|
|
|
f = darray_last(fs);
|
|
end = min(end, folio_end_pos(f));
|
|
last_folio_pos = folio_pos(f);
|
|
if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
|
|
if (end >= inode->v.i_size) {
|
|
folio_zero_range(f, 0, folio_size(f));
|
|
} else {
|
|
ret = bch2_read_single_folio(f, mapping);
|
|
if (ret)
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr);
|
|
if (ret)
|
|
goto out;
|
|
|
|
f_pos = pos;
|
|
f_offset = pos - folio_pos(darray_first(fs));
|
|
darray_for_each(fs, fi) {
|
|
ssize_t f_reserved;
|
|
|
|
f = *fi;
|
|
f_len = min(end, folio_end_pos(f)) - f_pos;
|
|
f_reserved = bch2_folio_reservation_get_partial(c, inode, f, &res, f_offset, f_len);
|
|
|
|
if (unlikely(f_reserved != f_len)) {
|
|
if (f_reserved < 0) {
|
|
if (f == darray_first(fs)) {
|
|
ret = f_reserved;
|
|
goto out;
|
|
}
|
|
|
|
folios_trunc(&fs, fi);
|
|
end = min(end, folio_end_pos(darray_last(fs)));
|
|
} else {
|
|
folios_trunc(&fs, fi + 1);
|
|
end = f_pos + f_reserved;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
f_pos = folio_end_pos(f);
|
|
f_offset = 0;
|
|
}
|
|
|
|
if (mapping_writably_mapped(mapping))
|
|
darray_for_each(fs, fi)
|
|
flush_dcache_folio(*fi);
|
|
|
|
f_pos = pos;
|
|
f_offset = pos - folio_pos(darray_first(fs));
|
|
darray_for_each(fs, fi) {
|
|
f = *fi;
|
|
f_len = min(end, folio_end_pos(f)) - f_pos;
|
|
f_copied = copy_folio_from_iter_atomic(f, f_offset, f_len, iter);
|
|
if (!f_copied) {
|
|
folios_trunc(&fs, fi);
|
|
break;
|
|
}
|
|
|
|
if (!folio_test_uptodate(f) &&
|
|
f_copied != folio_size(f) &&
|
|
pos + copied + f_copied < inode->v.i_size) {
|
|
iov_iter_revert(iter, f_copied);
|
|
folio_zero_range(f, 0, folio_size(f));
|
|
folios_trunc(&fs, fi);
|
|
break;
|
|
}
|
|
|
|
flush_dcache_folio(f);
|
|
copied += f_copied;
|
|
|
|
if (f_copied != f_len) {
|
|
folios_trunc(&fs, fi + 1);
|
|
break;
|
|
}
|
|
|
|
f_pos = folio_end_pos(f);
|
|
f_offset = 0;
|
|
}
|
|
|
|
if (!copied)
|
|
goto out;
|
|
|
|
end = pos + copied;
|
|
|
|
spin_lock(&inode->v.i_lock);
|
|
if (end > inode->v.i_size) {
|
|
BUG_ON(!inode_locked);
|
|
i_size_write(&inode->v, end);
|
|
}
|
|
spin_unlock(&inode->v.i_lock);
|
|
|
|
f_pos = pos;
|
|
f_offset = pos - folio_pos(darray_first(fs));
|
|
darray_for_each(fs, fi) {
|
|
f = *fi;
|
|
f_len = min(end, folio_end_pos(f)) - f_pos;
|
|
|
|
if (!folio_test_uptodate(f))
|
|
folio_mark_uptodate(f);
|
|
|
|
bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
|
|
|
|
f_pos = folio_end_pos(f);
|
|
f_offset = 0;
|
|
}
|
|
|
|
inode->ei_last_dirtied = (unsigned long) current;
|
|
out:
|
|
darray_for_each(fs, fi) {
|
|
folio_unlock(*fi);
|
|
folio_put(*fi);
|
|
}
|
|
|
|
/*
|
|
* If the last folio added to the mapping starts beyond current EOF, we
|
|
* performed a short write but left around at least one post-EOF folio.
|
|
* Clean up the mapping before we return.
|
|
*/
|
|
if (last_folio_pos >= inode->v.i_size)
|
|
truncate_pagecache(&inode->v, inode->v.i_size);
|
|
|
|
darray_exit(&fs);
|
|
bch2_folio_reservation_put(c, inode, &res);
|
|
|
|
return copied ?: ret;
|
|
}
|
|
|
|
static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
|
|
{
|
|
struct file *file = iocb->ki_filp;
|
|
struct address_space *mapping = file->f_mapping;
|
|
struct bch_inode_info *inode = file_bch_inode(file);
|
|
loff_t pos;
|
|
bool inode_locked = false;
|
|
ssize_t written = 0, written2 = 0, ret = 0;
|
|
|
|
/*
|
|
* We don't take the inode lock unless i_size will be changing. Folio
|
|
* locks provide exclusion with other writes, and the pagecache add lock
|
|
* provides exclusion with truncate and hole punching.
|
|
*
|
|
* There is one nasty corner case where atomicity would be broken
|
|
* without great care: when copying data from userspace to the page
|
|
* cache, we do that with faults disable - a page fault would recurse
|
|
* back into the filesystem, taking filesystem locks again, and
|
|
* deadlock; so it's done with faults disabled, and we fault in the user
|
|
* buffer when we aren't holding locks.
|
|
*
|
|
* If we do part of the write, but we then race and in the userspace
|
|
* buffer have been evicted and are no longer resident, then we have to
|
|
* drop our folio locks to re-fault them in, breaking write atomicity.
|
|
*
|
|
* To fix this, we restart the write from the start, if we weren't
|
|
* holding the inode lock.
|
|
*
|
|
* There is another wrinkle after that; if we restart the write from the
|
|
* start, and then get an unrecoverable error, we _cannot_ claim to
|
|
* userspace that we did not write data we actually did - so we must
|
|
* track (written2) the most we ever wrote.
|
|
*/
|
|
|
|
if ((iocb->ki_flags & IOCB_APPEND) ||
|
|
(iocb->ki_pos + iov_iter_count(iter) > i_size_read(&inode->v))) {
|
|
inode_lock(&inode->v);
|
|
inode_locked = true;
|
|
}
|
|
|
|
ret = generic_write_checks(iocb, iter);
|
|
if (ret <= 0)
|
|
goto unlock;
|
|
|
|
ret = file_remove_privs_flags(file, !inode_locked ? IOCB_NOWAIT : 0);
|
|
if (ret) {
|
|
if (!inode_locked) {
|
|
inode_lock(&inode->v);
|
|
inode_locked = true;
|
|
ret = file_remove_privs_flags(file, 0);
|
|
}
|
|
if (ret)
|
|
goto unlock;
|
|
}
|
|
|
|
ret = file_update_time(file);
|
|
if (ret)
|
|
goto unlock;
|
|
|
|
pos = iocb->ki_pos;
|
|
|
|
bch2_pagecache_add_get(inode);
|
|
|
|
if (!inode_locked &&
|
|
(iocb->ki_pos + iov_iter_count(iter) > i_size_read(&inode->v)))
|
|
goto get_inode_lock;
|
|
|
|
do {
|
|
unsigned offset = pos & (PAGE_SIZE - 1);
|
|
unsigned bytes = iov_iter_count(iter);
|
|
again:
|
|
/*
|
|
* Bring in the user page that we will copy from _first_.
|
|
* Otherwise there's a nasty deadlock on copying from the
|
|
* same page as we're writing to, without it being marked
|
|
* up-to-date.
|
|
*
|
|
* Not only is this an optimisation, but it is also required
|
|
* to check that the address is actually valid, when atomic
|
|
* usercopies are used, below.
|
|
*/
|
|
if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
|
|
bytes = min_t(unsigned long, iov_iter_count(iter),
|
|
PAGE_SIZE - offset);
|
|
|
|
if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
|
|
ret = -EFAULT;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (unlikely(bytes != iov_iter_count(iter) && !inode_locked))
|
|
goto get_inode_lock;
|
|
|
|
if (unlikely(fatal_signal_pending(current))) {
|
|
ret = -EINTR;
|
|
break;
|
|
}
|
|
|
|
ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes, inode_locked);
|
|
if (ret == -BCH_ERR_need_inode_lock)
|
|
goto get_inode_lock;
|
|
if (unlikely(ret < 0))
|
|
break;
|
|
|
|
cond_resched();
|
|
|
|
if (unlikely(ret == 0)) {
|
|
/*
|
|
* If we were unable to copy any data at all, we must
|
|
* fall back to a single segment length write.
|
|
*
|
|
* If we didn't fallback here, we could livelock
|
|
* because not all segments in the iov can be copied at
|
|
* once without a pagefault.
|
|
*/
|
|
bytes = min_t(unsigned long, PAGE_SIZE - offset,
|
|
iov_iter_single_seg_count(iter));
|
|
goto again;
|
|
}
|
|
pos += ret;
|
|
written += ret;
|
|
written2 = max(written, written2);
|
|
|
|
if (ret != bytes && !inode_locked)
|
|
goto get_inode_lock;
|
|
ret = 0;
|
|
|
|
balance_dirty_pages_ratelimited(mapping);
|
|
|
|
if (0) {
|
|
get_inode_lock:
|
|
bch2_pagecache_add_put(inode);
|
|
inode_lock(&inode->v);
|
|
inode_locked = true;
|
|
bch2_pagecache_add_get(inode);
|
|
|
|
iov_iter_revert(iter, written);
|
|
pos -= written;
|
|
written = 0;
|
|
ret = 0;
|
|
}
|
|
} while (iov_iter_count(iter));
|
|
bch2_pagecache_add_put(inode);
|
|
unlock:
|
|
if (inode_locked)
|
|
inode_unlock(&inode->v);
|
|
|
|
iocb->ki_pos += written;
|
|
|
|
ret = max(written, written2) ?: ret;
|
|
if (ret > 0)
|
|
ret = generic_write_sync(iocb, ret);
|
|
return ret;
|
|
}
|
|
|
|
ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *iter)
|
|
{
|
|
ssize_t ret = iocb->ki_flags & IOCB_DIRECT
|
|
? bch2_direct_write(iocb, iter)
|
|
: bch2_buffered_write(iocb, iter);
|
|
|
|
return bch2_err_class(ret);
|
|
}
|
|
|
|
void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)
|
|
{
|
|
bioset_exit(&c->writepage_bioset);
|
|
}
|
|
|
|
int bch2_fs_fs_io_buffered_init(struct bch_fs *c)
|
|
{
|
|
if (bioset_init(&c->writepage_bioset,
|
|
4, offsetof(struct bch_writepage_io, op.wbio.bio),
|
|
BIOSET_NEED_BVECS))
|
|
return -BCH_ERR_ENOMEM_writepage_bioset_init;
|
|
|
|
return 0;
|
|
}
|
|
|
|
#endif /* NO_BCACHEFS_FS */
|