forked from Minki/linux
blk-mq: abstract out helpers for allocating/freeing tag maps
Prep patch for adding an extra tag map for scheduler requests. Signed-off-by: Jens Axboe <axboe@fb.com> Reviewed-by: Bart Van Assche <bart.vanassche@sandisk.com> Reviewed-by: Omar Sandoval <osandov@fb.com>
This commit is contained in:
parent
4941115bef
commit
cc71a6f438
113
block/blk-mq.c
113
block/blk-mq.c
|
@ -1553,7 +1553,7 @@ run_queue:
|
||||||
return cookie;
|
return cookie;
|
||||||
}
|
}
|
||||||
|
|
||||||
void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
||||||
unsigned int hctx_idx)
|
unsigned int hctx_idx)
|
||||||
{
|
{
|
||||||
struct page *page;
|
struct page *page;
|
||||||
|
@ -1580,33 +1580,30 @@ void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
||||||
kmemleak_free(page_address(page));
|
kmemleak_free(page_address(page));
|
||||||
__free_pages(page, page->private);
|
__free_pages(page, page->private);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
|
||||||
|
{
|
||||||
kfree(tags->rqs);
|
kfree(tags->rqs);
|
||||||
|
tags->rqs = NULL;
|
||||||
|
|
||||||
blk_mq_free_tags(tags);
|
blk_mq_free_tags(tags);
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t order_to_size(unsigned int order)
|
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
|
||||||
{
|
unsigned int hctx_idx,
|
||||||
return (size_t)PAGE_SIZE << order;
|
unsigned int nr_tags,
|
||||||
}
|
unsigned int reserved_tags)
|
||||||
|
|
||||||
struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
|
|
||||||
unsigned int hctx_idx)
|
|
||||||
{
|
{
|
||||||
struct blk_mq_tags *tags;
|
struct blk_mq_tags *tags;
|
||||||
unsigned int i, j, entries_per_page, max_order = 4;
|
|
||||||
size_t rq_size, left;
|
|
||||||
|
|
||||||
tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
|
tags = blk_mq_init_tags(nr_tags, reserved_tags,
|
||||||
set->numa_node,
|
set->numa_node,
|
||||||
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
|
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
|
||||||
if (!tags)
|
if (!tags)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
INIT_LIST_HEAD(&tags->page_list);
|
tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
|
||||||
|
|
||||||
tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
|
|
||||||
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
|
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
|
||||||
set->numa_node);
|
set->numa_node);
|
||||||
if (!tags->rqs) {
|
if (!tags->rqs) {
|
||||||
|
@ -1614,15 +1611,31 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return tags;
|
||||||
|
}
|
||||||
|
|
||||||
|
static size_t order_to_size(unsigned int order)
|
||||||
|
{
|
||||||
|
return (size_t)PAGE_SIZE << order;
|
||||||
|
}
|
||||||
|
|
||||||
|
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
||||||
|
unsigned int hctx_idx, unsigned int depth)
|
||||||
|
{
|
||||||
|
unsigned int i, j, entries_per_page, max_order = 4;
|
||||||
|
size_t rq_size, left;
|
||||||
|
|
||||||
|
INIT_LIST_HEAD(&tags->page_list);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* rq_size is the size of the request plus driver payload, rounded
|
* rq_size is the size of the request plus driver payload, rounded
|
||||||
* to the cacheline size
|
* to the cacheline size
|
||||||
*/
|
*/
|
||||||
rq_size = round_up(sizeof(struct request) + set->cmd_size,
|
rq_size = round_up(sizeof(struct request) + set->cmd_size,
|
||||||
cache_line_size());
|
cache_line_size());
|
||||||
left = rq_size * set->queue_depth;
|
left = rq_size * depth;
|
||||||
|
|
||||||
for (i = 0; i < set->queue_depth; ) {
|
for (i = 0; i < depth; ) {
|
||||||
int this_order = max_order;
|
int this_order = max_order;
|
||||||
struct page *page;
|
struct page *page;
|
||||||
int to_do;
|
int to_do;
|
||||||
|
@ -1656,7 +1669,7 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
|
||||||
*/
|
*/
|
||||||
kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
|
kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
|
||||||
entries_per_page = order_to_size(this_order) / rq_size;
|
entries_per_page = order_to_size(this_order) / rq_size;
|
||||||
to_do = min(entries_per_page, set->queue_depth - i);
|
to_do = min(entries_per_page, depth - i);
|
||||||
left -= to_do * rq_size;
|
left -= to_do * rq_size;
|
||||||
for (j = 0; j < to_do; j++) {
|
for (j = 0; j < to_do; j++) {
|
||||||
tags->rqs[i] = p;
|
tags->rqs[i] = p;
|
||||||
|
@ -1673,11 +1686,11 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return tags;
|
return 0;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
blk_mq_free_rq_map(set, tags, hctx_idx);
|
blk_mq_free_rqs(set, tags, hctx_idx);
|
||||||
return NULL;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1869,6 +1882,33 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
|
||||||
|
{
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
|
||||||
|
set->queue_depth, set->reserved_tags);
|
||||||
|
if (!set->tags[hctx_idx])
|
||||||
|
return false;
|
||||||
|
|
||||||
|
ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
|
||||||
|
set->queue_depth);
|
||||||
|
if (!ret)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
blk_mq_free_rq_map(set->tags[hctx_idx]);
|
||||||
|
set->tags[hctx_idx] = NULL;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
|
||||||
|
unsigned int hctx_idx)
|
||||||
|
{
|
||||||
|
blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
|
||||||
|
blk_mq_free_rq_map(set->tags[hctx_idx]);
|
||||||
|
set->tags[hctx_idx] = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
static void blk_mq_map_swqueue(struct request_queue *q,
|
static void blk_mq_map_swqueue(struct request_queue *q,
|
||||||
const struct cpumask *online_mask)
|
const struct cpumask *online_mask)
|
||||||
{
|
{
|
||||||
|
@ -1897,16 +1937,14 @@ static void blk_mq_map_swqueue(struct request_queue *q,
|
||||||
|
|
||||||
hctx_idx = q->mq_map[i];
|
hctx_idx = q->mq_map[i];
|
||||||
/* unmapped hw queue can be remapped after CPU topo changed */
|
/* unmapped hw queue can be remapped after CPU topo changed */
|
||||||
if (!set->tags[hctx_idx]) {
|
if (!set->tags[hctx_idx] &&
|
||||||
set->tags[hctx_idx] = blk_mq_init_rq_map(set, hctx_idx);
|
!__blk_mq_alloc_rq_map(set, hctx_idx)) {
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If tags initialization fail for some hctx,
|
* If tags initialization fail for some hctx,
|
||||||
* that hctx won't be brought online. In this
|
* that hctx won't be brought online. In this
|
||||||
* case, remap the current ctx to hctx[0] which
|
* case, remap the current ctx to hctx[0] which
|
||||||
* is guaranteed to always have tags allocated
|
* is guaranteed to always have tags allocated
|
||||||
*/
|
*/
|
||||||
if (!set->tags[hctx_idx])
|
|
||||||
q->mq_map[i] = 0;
|
q->mq_map[i] = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1930,10 +1968,9 @@ static void blk_mq_map_swqueue(struct request_queue *q,
|
||||||
* fallback in case of a new remap fails
|
* fallback in case of a new remap fails
|
||||||
* allocation
|
* allocation
|
||||||
*/
|
*/
|
||||||
if (i && set->tags[i]) {
|
if (i && set->tags[i])
|
||||||
blk_mq_free_rq_map(set, set->tags[i], i);
|
blk_mq_free_map_and_requests(set, i);
|
||||||
set->tags[i] = NULL;
|
|
||||||
}
|
|
||||||
hctx->tags = NULL;
|
hctx->tags = NULL;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -2100,10 +2137,8 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
|
||||||
struct blk_mq_hw_ctx *hctx = hctxs[j];
|
struct blk_mq_hw_ctx *hctx = hctxs[j];
|
||||||
|
|
||||||
if (hctx) {
|
if (hctx) {
|
||||||
if (hctx->tags) {
|
if (hctx->tags)
|
||||||
blk_mq_free_rq_map(set, hctx->tags, j);
|
blk_mq_free_map_and_requests(set, j);
|
||||||
set->tags[j] = NULL;
|
|
||||||
}
|
|
||||||
blk_mq_exit_hctx(q, set, hctx, j);
|
blk_mq_exit_hctx(q, set, hctx, j);
|
||||||
free_cpumask_var(hctx->cpumask);
|
free_cpumask_var(hctx->cpumask);
|
||||||
kobject_put(&hctx->kobj);
|
kobject_put(&hctx->kobj);
|
||||||
|
@ -2299,17 +2334,15 @@ static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
for (i = 0; i < set->nr_hw_queues; i++) {
|
for (i = 0; i < set->nr_hw_queues; i++)
|
||||||
set->tags[i] = blk_mq_init_rq_map(set, i);
|
if (!__blk_mq_alloc_rq_map(set, i))
|
||||||
if (!set->tags[i])
|
|
||||||
goto out_unwind;
|
goto out_unwind;
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
out_unwind:
|
out_unwind:
|
||||||
while (--i >= 0)
|
while (--i >= 0)
|
||||||
blk_mq_free_rq_map(set, set->tags[i], i);
|
blk_mq_free_rq_map(set->tags[i]);
|
||||||
|
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
@ -2433,10 +2466,8 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
for (i = 0; i < nr_cpu_ids; i++) {
|
for (i = 0; i < nr_cpu_ids; i++)
|
||||||
if (set->tags[i])
|
blk_mq_free_map_and_requests(set, i);
|
||||||
blk_mq_free_rq_map(set, set->tags[i], i);
|
|
||||||
}
|
|
||||||
|
|
||||||
kfree(set->mq_map);
|
kfree(set->mq_map);
|
||||||
set->mq_map = NULL;
|
set->mq_map = NULL;
|
||||||
|
|
|
@ -37,17 +37,21 @@ void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
|
||||||
/*
|
/*
|
||||||
* Internal helpers for allocating/freeing the request map
|
* Internal helpers for allocating/freeing the request map
|
||||||
*/
|
*/
|
||||||
void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
||||||
unsigned int hctx_idx);
|
|
||||||
struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
|
|
||||||
unsigned int hctx_idx);
|
unsigned int hctx_idx);
|
||||||
|
void blk_mq_free_rq_map(struct blk_mq_tags *tags);
|
||||||
|
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
|
||||||
|
unsigned int hctx_idx,
|
||||||
|
unsigned int nr_tags,
|
||||||
|
unsigned int reserved_tags);
|
||||||
|
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
|
||||||
|
unsigned int hctx_idx, unsigned int depth);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Internal helpers for request insertion into sw queues
|
* Internal helpers for request insertion into sw queues
|
||||||
*/
|
*/
|
||||||
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
|
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
|
||||||
bool at_head);
|
bool at_head);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* CPU hotplug helpers
|
* CPU hotplug helpers
|
||||||
*/
|
*/
|
||||||
|
|
Loading…
Reference in New Issue
Block a user