leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

block: inline fast path of driver tag allocation

Browse Source 
If we don't use an IO scheduler or have shared tags, then we don't need
to call into this external function at all. This saves ~2% for such
a setup.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2021-10-13 08:28:14 -06:00
parent d92ca9d834
commit a808a9d545
2 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
block/blk-mq.c
View File

@ -1324,7 +1324,7 @@ struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
return data.rq; return data.rq;
} }
static bool __blk_mq_get_driver_tag(struct request *rq) static bool __blk_mq_alloc_driver_tag(struct request *rq)
{ {
struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
@ -1348,11 +1348,9 @@ static bool __blk_mq_get_driver_tag(struct request *rq)
return true; return true;
} }
bool blk_mq_get_driver_tag(struct request *rq) bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq)
{ {
struct blk_mq_hw_ctx *hctx = rq->mq_hctx; if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_alloc_driver_tag(rq))
if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
return false; return false;
if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) && if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&

15
block/blk-mq.h
View File

@ -258,7 +258,20 @@ static inline void blk_mq_put_driver_tag(struct request *rq)
__blk_mq_put_driver_tag(rq->mq_hctx, rq); __blk_mq_put_driver_tag(rq->mq_hctx, rq);
} }
bool blk_mq_get_driver_tag(struct request *rq); bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq);
static inline bool blk_mq_get_driver_tag(struct request *rq)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
if (rq->tag != BLK_MQ_NO_TAG &&
!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
hctx->tags->rqs[rq->tag] = rq;
return true;
}
return __blk_mq_get_driver_tag(hctx, rq);
}
static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap) static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap)
{ {