mirror of
https://github.com/torvalds/linux.git
synced 2024-12-13 22:53:20 +00:00
8374cfe647
being split acorss files. - Improve DM core's BLK_STS_DM_REQUEUE and BLK_STS_AGAIN handling. - Optimize DM core's more common bio splitting by eliminating the use of bio cloning with bio_split+bio_chain. Shift that cloning cost to the relatively unlikely dm_io requeue case that only occurs during error handling. Introduces dm_io_rewind() that will clone a bio that reflects the subset of the original bio that must be requeued. - Remove DM core's dm_table_get_num_targets() wrapper and audit all dm_table_get_target() callers. - Fix potential for OOM with DM writecache target by setting a default MAX_WRITEBACK_JOBS (set to 256MiB or 1/16 of total system memory, whichever is smaller). - Fix DM writecache target's stats that are reported through DM-specific table info. - Fix use-after-free crash in dm_sm_register_threshold_callback(). - Refine DM core's Persistent Reservation handling in preparation for broader work Mike Christie is doing to add compatibility with Microsoft Windows Failover Cluster. - Fix various KASAN reported bugs in the DM raid target. - Fix DM raid target crash due to md_handle_request() bio splitting that recurses to block core without properly initializing the bio's bi_dev. - Fix some code comment typos and fix some Documentation formatting. -----BEGIN PGP SIGNATURE----- iQEzBAABCAAdFiEEJfWUX4UqZ4x1O2wixSPxCi2dA1oFAmLoAAUACgkQxSPxCi2d A1rFUAf/RnLkzNPS1QJ1uVbiiW64zQUD2o5U8kAliOZxoXQ45U+CgL22a5VaT2R+ rI9+YSg/VX9YkdJwB6I+y7VRWjUCO3NfYOfQUX5NS8GfcPQQn2Zp+jy3t8VnjjXN 5+8Ylu5tX1+QSQiBB9JvIgiK71rSorT6H+KF6bZypL7te5kj39hDaRbmh40VOMOY QWfs8DmyJft9IHPG7Mku/rFJbdVJph3f31IoqUOoGTOKoDPDUDezhCVHi2vpFcV+ S+iCKkeXmP6eUdCnNzBreYPTcP9OtCvkZEPhFg4lh+jyhjyFq1o0B8G5cRo5jvgr GcN1GUT040sSqNXFquA4UU6RGvaxlg== =gBRJ -----END PGP SIGNATURE----- Merge tag 'for-6.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm Pull device mapper updates from Mike Snitzer: - Refactor DM core's mempool allocation so that it clearer by not being split acorss files. - Improve DM core's BLK_STS_DM_REQUEUE and BLK_STS_AGAIN handling. - Optimize DM core's more common bio splitting by eliminating the use of bio cloning with bio_split+bio_chain. Shift that cloning cost to the relatively unlikely dm_io requeue case that only occurs during error handling. Introduces dm_io_rewind() that will clone a bio that reflects the subset of the original bio that must be requeued. - Remove DM core's dm_table_get_num_targets() wrapper and audit all dm_table_get_target() callers. - Fix potential for OOM with DM writecache target by setting a default MAX_WRITEBACK_JOBS (set to 256MiB or 1/16 of total system memory, whichever is smaller). - Fix DM writecache target's stats that are reported through DM-specific table info. - Fix use-after-free crash in dm_sm_register_threshold_callback(). - Refine DM core's Persistent Reservation handling in preparation for broader work Mike Christie is doing to add compatibility with Microsoft Windows Failover Cluster. - Fix various KASAN reported bugs in the DM raid target. - Fix DM raid target crash due to md_handle_request() bio splitting that recurses to block core without properly initializing the bio's bi_dev. - Fix some code comment typos and fix some Documentation formatting. * tag 'for-6.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: (29 commits) dm: fix dm-raid crash if md_handle_request() splits bio dm raid: fix address sanitizer warning in raid_resume dm raid: fix address sanitizer warning in raid_status dm: Start pr_preempt from the same starting path dm: Fix PR release handling for non All Registrants dm: Start pr_reserve from the same starting path dm: Allow dm_call_pr to be used for path searches dm: return early from dm_pr_call() if DM device is suspended dm thin: fix use-after-free crash in dm_sm_register_threshold_callback dm writecache: count number of blocks discarded, not number of discard bios dm writecache: count number of blocks written, not number of write bios dm writecache: count number of blocks read, not number of read bios dm writecache: return void from functions dm kcopyd: use __GFP_HIGHMEM when allocating pages dm writecache: set a default MAX_WRITEBACK_JOBS Documentation: dm writecache: Render status list as list Documentation: dm writecache: add blank line before optional parameters dm snapshot: fix typo in snapshot_map() comment dm raid: remove redundant "the" in parse_raid_params() comment dm cache: fix typo in 2 comment blocks ...
649 lines
16 KiB
C
649 lines
16 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
|
|
*/
|
|
|
|
#include <linux/blkdev.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/sched/mm.h>
|
|
#include <linux/slab.h>
|
|
|
|
#include "dm-core.h"
|
|
|
|
#define DM_MSG_PREFIX "zone"
|
|
|
|
#define DM_ZONE_INVALID_WP_OFST UINT_MAX
|
|
|
|
/*
|
|
* For internal zone reports bypassing the top BIO submission path.
|
|
*/
|
|
static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,
|
|
sector_t sector, unsigned int nr_zones,
|
|
report_zones_cb cb, void *data)
|
|
{
|
|
struct gendisk *disk = md->disk;
|
|
int ret;
|
|
struct dm_report_zones_args args = {
|
|
.next_sector = sector,
|
|
.orig_data = data,
|
|
.orig_cb = cb,
|
|
};
|
|
|
|
do {
|
|
struct dm_target *tgt;
|
|
|
|
tgt = dm_table_find_target(t, args.next_sector);
|
|
if (WARN_ON_ONCE(!tgt->type->report_zones))
|
|
return -EIO;
|
|
|
|
args.tgt = tgt;
|
|
ret = tgt->type->report_zones(tgt, &args,
|
|
nr_zones - args.zone_idx);
|
|
if (ret < 0)
|
|
return ret;
|
|
} while (args.zone_idx < nr_zones &&
|
|
args.next_sector < get_capacity(disk));
|
|
|
|
return args.zone_idx;
|
|
}
|
|
|
|
/*
|
|
* User facing dm device block device report zone operation. This calls the
|
|
* report_zones operation for each target of a device table. This operation is
|
|
* generally implemented by targets using dm_report_zones().
|
|
*/
|
|
int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
|
|
unsigned int nr_zones, report_zones_cb cb, void *data)
|
|
{
|
|
struct mapped_device *md = disk->private_data;
|
|
struct dm_table *map;
|
|
int srcu_idx, ret;
|
|
|
|
if (dm_suspended_md(md))
|
|
return -EAGAIN;
|
|
|
|
map = dm_get_live_table(md, &srcu_idx);
|
|
if (!map)
|
|
return -EIO;
|
|
|
|
ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);
|
|
|
|
dm_put_live_table(md, srcu_idx);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,
|
|
void *data)
|
|
{
|
|
struct dm_report_zones_args *args = data;
|
|
sector_t sector_diff = args->tgt->begin - args->start;
|
|
|
|
/*
|
|
* Ignore zones beyond the target range.
|
|
*/
|
|
if (zone->start >= args->start + args->tgt->len)
|
|
return 0;
|
|
|
|
/*
|
|
* Remap the start sector and write pointer position of the zone
|
|
* to match its position in the target range.
|
|
*/
|
|
zone->start += sector_diff;
|
|
if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
|
|
if (zone->cond == BLK_ZONE_COND_FULL)
|
|
zone->wp = zone->start + zone->len;
|
|
else if (zone->cond == BLK_ZONE_COND_EMPTY)
|
|
zone->wp = zone->start;
|
|
else
|
|
zone->wp += sector_diff;
|
|
}
|
|
|
|
args->next_sector = zone->start + zone->len;
|
|
return args->orig_cb(zone, args->zone_idx++, args->orig_data);
|
|
}
|
|
|
|
/*
|
|
* Helper for drivers of zoned targets to implement struct target_type
|
|
* report_zones operation.
|
|
*/
|
|
int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,
|
|
struct dm_report_zones_args *args, unsigned int nr_zones)
|
|
{
|
|
/*
|
|
* Set the target mapping start sector first so that
|
|
* dm_report_zones_cb() can correctly remap zone information.
|
|
*/
|
|
args->start = start;
|
|
|
|
return blkdev_report_zones(bdev, sector, nr_zones,
|
|
dm_report_zones_cb, args);
|
|
}
|
|
EXPORT_SYMBOL_GPL(dm_report_zones);
|
|
|
|
bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
|
|
{
|
|
struct request_queue *q = md->queue;
|
|
|
|
if (!blk_queue_is_zoned(q))
|
|
return false;
|
|
|
|
switch (bio_op(bio)) {
|
|
case REQ_OP_WRITE_ZEROES:
|
|
case REQ_OP_WRITE:
|
|
return !op_is_flush(bio->bi_opf) && bio_sectors(bio);
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void dm_cleanup_zoned_dev(struct mapped_device *md)
|
|
{
|
|
if (md->disk) {
|
|
kfree(md->disk->conv_zones_bitmap);
|
|
md->disk->conv_zones_bitmap = NULL;
|
|
kfree(md->disk->seq_zones_wlock);
|
|
md->disk->seq_zones_wlock = NULL;
|
|
}
|
|
|
|
kvfree(md->zwp_offset);
|
|
md->zwp_offset = NULL;
|
|
md->nr_zones = 0;
|
|
}
|
|
|
|
static unsigned int dm_get_zone_wp_offset(struct blk_zone *zone)
|
|
{
|
|
switch (zone->cond) {
|
|
case BLK_ZONE_COND_IMP_OPEN:
|
|
case BLK_ZONE_COND_EXP_OPEN:
|
|
case BLK_ZONE_COND_CLOSED:
|
|
return zone->wp - zone->start;
|
|
case BLK_ZONE_COND_FULL:
|
|
return zone->len;
|
|
case BLK_ZONE_COND_EMPTY:
|
|
case BLK_ZONE_COND_NOT_WP:
|
|
case BLK_ZONE_COND_OFFLINE:
|
|
case BLK_ZONE_COND_READONLY:
|
|
default:
|
|
/*
|
|
* Conventional, offline and read-only zones do not have a valid
|
|
* write pointer. Use 0 as for an empty zone.
|
|
*/
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx,
|
|
void *data)
|
|
{
|
|
struct mapped_device *md = data;
|
|
struct gendisk *disk = md->disk;
|
|
|
|
switch (zone->type) {
|
|
case BLK_ZONE_TYPE_CONVENTIONAL:
|
|
if (!disk->conv_zones_bitmap) {
|
|
disk->conv_zones_bitmap =
|
|
kcalloc(BITS_TO_LONGS(disk->nr_zones),
|
|
sizeof(unsigned long), GFP_NOIO);
|
|
if (!disk->conv_zones_bitmap)
|
|
return -ENOMEM;
|
|
}
|
|
set_bit(idx, disk->conv_zones_bitmap);
|
|
break;
|
|
case BLK_ZONE_TYPE_SEQWRITE_REQ:
|
|
case BLK_ZONE_TYPE_SEQWRITE_PREF:
|
|
if (!disk->seq_zones_wlock) {
|
|
disk->seq_zones_wlock =
|
|
kcalloc(BITS_TO_LONGS(disk->nr_zones),
|
|
sizeof(unsigned long), GFP_NOIO);
|
|
if (!disk->seq_zones_wlock)
|
|
return -ENOMEM;
|
|
}
|
|
if (!md->zwp_offset) {
|
|
md->zwp_offset =
|
|
kvcalloc(disk->nr_zones, sizeof(unsigned int),
|
|
GFP_KERNEL);
|
|
if (!md->zwp_offset)
|
|
return -ENOMEM;
|
|
}
|
|
md->zwp_offset[idx] = dm_get_zone_wp_offset(zone);
|
|
|
|
break;
|
|
default:
|
|
DMERR("Invalid zone type 0x%x at sectors %llu",
|
|
(int)zone->type, zone->start);
|
|
return -ENODEV;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Revalidate the zones of a mapped device to initialize resource necessary
|
|
* for zone append emulation. Note that we cannot simply use the block layer
|
|
* blk_revalidate_disk_zones() function here as the mapped device is suspended
|
|
* (this is called from __bind() context).
|
|
*/
|
|
static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t)
|
|
{
|
|
struct gendisk *disk = md->disk;
|
|
unsigned int noio_flag;
|
|
int ret;
|
|
|
|
/*
|
|
* Check if something changed. If yes, cleanup the current resources
|
|
* and reallocate everything.
|
|
*/
|
|
if (!disk->nr_zones || disk->nr_zones != md->nr_zones)
|
|
dm_cleanup_zoned_dev(md);
|
|
if (md->nr_zones)
|
|
return 0;
|
|
|
|
/*
|
|
* Scan all zones to initialize everything. Ensure that all vmalloc
|
|
* operations in this context are done as if GFP_NOIO was specified.
|
|
*/
|
|
noio_flag = memalloc_noio_save();
|
|
ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones,
|
|
dm_zone_revalidate_cb, md);
|
|
memalloc_noio_restore(noio_flag);
|
|
if (ret < 0)
|
|
goto err;
|
|
if (ret != disk->nr_zones) {
|
|
ret = -EIO;
|
|
goto err;
|
|
}
|
|
|
|
md->nr_zones = disk->nr_zones;
|
|
|
|
return 0;
|
|
|
|
err:
|
|
DMERR("Revalidate zones failed %d", ret);
|
|
dm_cleanup_zoned_dev(md);
|
|
return ret;
|
|
}
|
|
|
|
static int device_not_zone_append_capable(struct dm_target *ti,
|
|
struct dm_dev *dev, sector_t start,
|
|
sector_t len, void *data)
|
|
{
|
|
return !bdev_is_zoned(dev->bdev);
|
|
}
|
|
|
|
static bool dm_table_supports_zone_append(struct dm_table *t)
|
|
{
|
|
for (unsigned int i = 0; i < t->num_targets; i++) {
|
|
struct dm_target *ti = dm_table_get_target(t, i);
|
|
|
|
if (ti->emulate_zone_append)
|
|
return false;
|
|
|
|
if (!ti->type->iterate_devices ||
|
|
ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q)
|
|
{
|
|
struct mapped_device *md = t->md;
|
|
|
|
/*
|
|
* For a zoned target, the number of zones should be updated for the
|
|
* correct value to be exposed in sysfs queue/nr_zones.
|
|
*/
|
|
WARN_ON_ONCE(queue_is_mq(q));
|
|
md->disk->nr_zones = bdev_nr_zones(md->disk->part0);
|
|
|
|
/* Check if zone append is natively supported */
|
|
if (dm_table_supports_zone_append(t)) {
|
|
clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
|
|
dm_cleanup_zoned_dev(md);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Mark the mapped device as needing zone append emulation and
|
|
* initialize the emulation resources once the capacity is set.
|
|
*/
|
|
set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
|
|
if (!get_capacity(md->disk))
|
|
return 0;
|
|
|
|
return dm_revalidate_zones(md, t);
|
|
}
|
|
|
|
static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx,
|
|
void *data)
|
|
{
|
|
unsigned int *wp_offset = data;
|
|
|
|
*wp_offset = dm_get_zone_wp_offset(zone);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno,
|
|
unsigned int *wp_ofst)
|
|
{
|
|
sector_t sector = zno * bdev_zone_sectors(md->disk->part0);
|
|
unsigned int noio_flag;
|
|
struct dm_table *t;
|
|
int srcu_idx, ret;
|
|
|
|
t = dm_get_live_table(md, &srcu_idx);
|
|
if (!t)
|
|
return -EIO;
|
|
|
|
/*
|
|
* Ensure that all memory allocations in this context are done as if
|
|
* GFP_NOIO was specified.
|
|
*/
|
|
noio_flag = memalloc_noio_save();
|
|
ret = dm_blk_do_report_zones(md, t, sector, 1,
|
|
dm_update_zone_wp_offset_cb, wp_ofst);
|
|
memalloc_noio_restore(noio_flag);
|
|
|
|
dm_put_live_table(md, srcu_idx);
|
|
|
|
if (ret != 1)
|
|
return -EIO;
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct orig_bio_details {
|
|
enum req_op op;
|
|
unsigned int nr_sectors;
|
|
};
|
|
|
|
/*
|
|
* First phase of BIO mapping for targets with zone append emulation:
|
|
* check all BIO that change a zone writer pointer and change zone
|
|
* append operations into regular write operations.
|
|
*/
|
|
static bool dm_zone_map_bio_begin(struct mapped_device *md,
|
|
unsigned int zno, struct bio *clone)
|
|
{
|
|
sector_t zsectors = bdev_zone_sectors(md->disk->part0);
|
|
unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
|
|
|
|
/*
|
|
* If the target zone is in an error state, recover by inspecting the
|
|
* zone to get its current write pointer position. Note that since the
|
|
* target zone is already locked, a BIO issuing context should never
|
|
* see the zone write in the DM_ZONE_UPDATING_WP_OFST state.
|
|
*/
|
|
if (zwp_offset == DM_ZONE_INVALID_WP_OFST) {
|
|
if (dm_update_zone_wp_offset(md, zno, &zwp_offset))
|
|
return false;
|
|
WRITE_ONCE(md->zwp_offset[zno], zwp_offset);
|
|
}
|
|
|
|
switch (bio_op(clone)) {
|
|
case REQ_OP_ZONE_RESET:
|
|
case REQ_OP_ZONE_FINISH:
|
|
return true;
|
|
case REQ_OP_WRITE_ZEROES:
|
|
case REQ_OP_WRITE:
|
|
/* Writes must be aligned to the zone write pointer */
|
|
if ((clone->bi_iter.bi_sector & (zsectors - 1)) != zwp_offset)
|
|
return false;
|
|
break;
|
|
case REQ_OP_ZONE_APPEND:
|
|
/*
|
|
* Change zone append operations into a non-mergeable regular
|
|
* writes directed at the current write pointer position of the
|
|
* target zone.
|
|
*/
|
|
clone->bi_opf = REQ_OP_WRITE | REQ_NOMERGE |
|
|
(clone->bi_opf & (~REQ_OP_MASK));
|
|
clone->bi_iter.bi_sector += zwp_offset;
|
|
break;
|
|
default:
|
|
DMWARN_LIMIT("Invalid BIO operation");
|
|
return false;
|
|
}
|
|
|
|
/* Cannot write to a full zone */
|
|
if (zwp_offset >= zsectors)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
/*
|
|
* Second phase of BIO mapping for targets with zone append emulation:
|
|
* update the zone write pointer offset array to account for the additional
|
|
* data written to a zone. Note that at this point, the remapped clone BIO
|
|
* may already have completed, so we do not touch it.
|
|
*/
|
|
static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int zno,
|
|
struct orig_bio_details *orig_bio_details,
|
|
unsigned int nr_sectors)
|
|
{
|
|
unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
|
|
|
|
/* The clone BIO may already have been completed and failed */
|
|
if (zwp_offset == DM_ZONE_INVALID_WP_OFST)
|
|
return BLK_STS_IOERR;
|
|
|
|
/* Update the zone wp offset */
|
|
switch (orig_bio_details->op) {
|
|
case REQ_OP_ZONE_RESET:
|
|
WRITE_ONCE(md->zwp_offset[zno], 0);
|
|
return BLK_STS_OK;
|
|
case REQ_OP_ZONE_FINISH:
|
|
WRITE_ONCE(md->zwp_offset[zno],
|
|
bdev_zone_sectors(md->disk->part0));
|
|
return BLK_STS_OK;
|
|
case REQ_OP_WRITE_ZEROES:
|
|
case REQ_OP_WRITE:
|
|
WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
|
|
return BLK_STS_OK;
|
|
case REQ_OP_ZONE_APPEND:
|
|
/*
|
|
* Check that the target did not truncate the write operation
|
|
* emulating a zone append.
|
|
*/
|
|
if (nr_sectors != orig_bio_details->nr_sectors) {
|
|
DMWARN_LIMIT("Truncated write for zone append");
|
|
return BLK_STS_IOERR;
|
|
}
|
|
WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
|
|
return BLK_STS_OK;
|
|
default:
|
|
DMWARN_LIMIT("Invalid BIO operation");
|
|
return BLK_STS_IOERR;
|
|
}
|
|
}
|
|
|
|
static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno,
|
|
struct bio *clone)
|
|
{
|
|
if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)))
|
|
return;
|
|
|
|
wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE);
|
|
bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED);
|
|
}
|
|
|
|
static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno,
|
|
struct bio *clone)
|
|
{
|
|
if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
|
|
return;
|
|
|
|
WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock));
|
|
clear_bit_unlock(zno, disk->seq_zones_wlock);
|
|
smp_mb__after_atomic();
|
|
wake_up_bit(disk->seq_zones_wlock, zno);
|
|
|
|
bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED);
|
|
}
|
|
|
|
static bool dm_need_zone_wp_tracking(struct bio *bio)
|
|
{
|
|
/*
|
|
* Special processing is not needed for operations that do not need the
|
|
* zone write lock, that is, all operations that target conventional
|
|
* zones and all operations that do not modify directly a sequential
|
|
* zone write pointer.
|
|
*/
|
|
if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
|
|
return false;
|
|
switch (bio_op(bio)) {
|
|
case REQ_OP_WRITE_ZEROES:
|
|
case REQ_OP_WRITE:
|
|
case REQ_OP_ZONE_RESET:
|
|
case REQ_OP_ZONE_FINISH:
|
|
case REQ_OP_ZONE_APPEND:
|
|
return bio_zone_is_seq(bio);
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Special IO mapping for targets needing zone append emulation.
|
|
*/
|
|
int dm_zone_map_bio(struct dm_target_io *tio)
|
|
{
|
|
struct dm_io *io = tio->io;
|
|
struct dm_target *ti = tio->ti;
|
|
struct mapped_device *md = io->md;
|
|
struct bio *clone = &tio->clone;
|
|
struct orig_bio_details orig_bio_details;
|
|
unsigned int zno;
|
|
blk_status_t sts;
|
|
int r;
|
|
|
|
/*
|
|
* IOs that do not change a zone write pointer do not need
|
|
* any additional special processing.
|
|
*/
|
|
if (!dm_need_zone_wp_tracking(clone))
|
|
return ti->type->map(ti, clone);
|
|
|
|
/* Lock the target zone */
|
|
zno = bio_zone_no(clone);
|
|
dm_zone_lock(md->disk, zno, clone);
|
|
|
|
orig_bio_details.nr_sectors = bio_sectors(clone);
|
|
orig_bio_details.op = bio_op(clone);
|
|
|
|
/*
|
|
* Check that the bio and the target zone write pointer offset are
|
|
* both valid, and if the bio is a zone append, remap it to a write.
|
|
*/
|
|
if (!dm_zone_map_bio_begin(md, zno, clone)) {
|
|
dm_zone_unlock(md->disk, zno, clone);
|
|
return DM_MAPIO_KILL;
|
|
}
|
|
|
|
/* Let the target do its work */
|
|
r = ti->type->map(ti, clone);
|
|
switch (r) {
|
|
case DM_MAPIO_SUBMITTED:
|
|
/*
|
|
* The target submitted the clone BIO. The target zone will
|
|
* be unlocked on completion of the clone.
|
|
*/
|
|
sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
|
|
*tio->len_ptr);
|
|
break;
|
|
case DM_MAPIO_REMAPPED:
|
|
/*
|
|
* The target only remapped the clone BIO. In case of error,
|
|
* unlock the target zone here as the clone will not be
|
|
* submitted.
|
|
*/
|
|
sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
|
|
*tio->len_ptr);
|
|
if (sts != BLK_STS_OK)
|
|
dm_zone_unlock(md->disk, zno, clone);
|
|
break;
|
|
case DM_MAPIO_REQUEUE:
|
|
case DM_MAPIO_KILL:
|
|
default:
|
|
dm_zone_unlock(md->disk, zno, clone);
|
|
sts = BLK_STS_IOERR;
|
|
break;
|
|
}
|
|
|
|
if (sts != BLK_STS_OK)
|
|
return DM_MAPIO_KILL;
|
|
|
|
return r;
|
|
}
|
|
|
|
/*
|
|
* IO completion callback called from clone_endio().
|
|
*/
|
|
void dm_zone_endio(struct dm_io *io, struct bio *clone)
|
|
{
|
|
struct mapped_device *md = io->md;
|
|
struct gendisk *disk = md->disk;
|
|
struct bio *orig_bio = io->orig_bio;
|
|
unsigned int zwp_offset;
|
|
unsigned int zno;
|
|
|
|
/*
|
|
* For targets that do not emulate zone append, we only need to
|
|
* handle native zone-append bios.
|
|
*/
|
|
if (!dm_emulate_zone_append(md)) {
|
|
/*
|
|
* Get the offset within the zone of the written sector
|
|
* and add that to the original bio sector position.
|
|
*/
|
|
if (clone->bi_status == BLK_STS_OK &&
|
|
bio_op(clone) == REQ_OP_ZONE_APPEND) {
|
|
sector_t mask =
|
|
(sector_t)bdev_zone_sectors(disk->part0) - 1;
|
|
|
|
orig_bio->bi_iter.bi_sector +=
|
|
clone->bi_iter.bi_sector & mask;
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* For targets that do emulate zone append, if the clone BIO does not
|
|
* own the target zone write lock, we have nothing to do.
|
|
*/
|
|
if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
|
|
return;
|
|
|
|
zno = bio_zone_no(orig_bio);
|
|
|
|
if (clone->bi_status != BLK_STS_OK) {
|
|
/*
|
|
* BIOs that modify a zone write pointer may leave the zone
|
|
* in an unknown state in case of failure (e.g. the write
|
|
* pointer was only partially advanced). In this case, set
|
|
* the target zone write pointer as invalid unless it is
|
|
* already being updated.
|
|
*/
|
|
WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST);
|
|
} else if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
|
|
/*
|
|
* Get the written sector for zone append operation that were
|
|
* emulated using regular write operations.
|
|
*/
|
|
zwp_offset = READ_ONCE(md->zwp_offset[zno]);
|
|
if (WARN_ON_ONCE(zwp_offset < bio_sectors(orig_bio)))
|
|
WRITE_ONCE(md->zwp_offset[zno],
|
|
DM_ZONE_INVALID_WP_OFST);
|
|
else
|
|
orig_bio->bi_iter.bi_sector +=
|
|
zwp_offset - bio_sectors(orig_bio);
|
|
}
|
|
|
|
dm_zone_unlock(disk, zno, clone);
|
|
}
|