Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-03 09:31:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

rnbd/rtrs-clt: Query and use max_segments from rtrs-clt.

Browse Source 
With fast memory registration on write request, rnbd-clt
can do bigger IO without split. rnbd-clt now can query
rtrs-clt to get the max_segments, instead of using
BMAX_SEGMENTS.

BMAX_SEGMENTS is not longer needed, so remove it.

Link: https://lore.kernel.org/r/20210621055340.11789-6-jinpu.wang@ionos.com
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Reviewed-by: Md Haris Iqbal <haris.iqbal@cloud.ionos.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
This commit is contained in:
Jack Wang 2021-06-21 07:53:40 +02:00 committed by Jason Gunthorpe
parent 6fc4559650
commit 7404bddeb4
4 changed files with 13 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
drivers/block/rnbd/rnbd-clt.c
View File

@ -92,7 +92,7 @@ static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev,
dev->fua = !!(rsp->cache_policy & RNBD_FUA); dev->fua = !!(rsp->cache_policy & RNBD_FUA);
dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE; dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE;
dev->max_segments = BMAX_SEGMENTS; dev->max_segments = sess->max_segments;
return 0; return 0;
} }
@ -1292,7 +1292,7 @@ find_and_get_or_create_sess(const char *sessname,
sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname, sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,
paths, path_cnt, port_nr, paths, path_cnt, port_nr,
0, /* Do not use pdu of rtrs */ 0, /* Do not use pdu of rtrs */
RECONNECT_DELAY, BMAX_SEGMENTS, RECONNECT_DELAY,
MAX_RECONNECTS, nr_poll_queues); MAX_RECONNECTS, nr_poll_queues);
if (IS_ERR(sess->rtrs)) { if (IS_ERR(sess->rtrs)) {
err = PTR_ERR(sess->rtrs); err = PTR_ERR(sess->rtrs);
@ -1306,6 +1306,7 @@ find_and_get_or_create_sess(const char *sessname,
sess->max_io_size = attrs.max_io_size; sess->max_io_size = attrs.max_io_size;
sess->queue_depth = attrs.queue_depth; sess->queue_depth = attrs.queue_depth;
sess->nr_poll_queues = nr_poll_queues; sess->nr_poll_queues = nr_poll_queues;
sess->max_segments = attrs.max_segments;
err = setup_mq_tags(sess); err = setup_mq_tags(sess);
if (err) if (err)

5
drivers/block/rnbd/rnbd-clt.h
View File

@ -20,10 +20,6 @@
#include "rnbd-proto.h" #include "rnbd-proto.h"
#include "rnbd-log.h" #include "rnbd-log.h"
/* Max. number of segments per IO request, Mellanox Connect X ~ Connect X5,
* choose minimial 30 for all, minus 1 for internal protocol, so 29.
*/
#define BMAX_SEGMENTS 29
/* time in seconds between reconnect tries, default to 30 s */ /* time in seconds between reconnect tries, default to 30 s */
#define RECONNECT_DELAY 30 #define RECONNECT_DELAY 30
/* /*
@ -89,6 +85,7 @@ struct rnbd_clt_session {
atomic_t busy; atomic_t busy;
size_t queue_depth; size_t queue_depth;
u32 max_io_size; u32 max_io_size;
u32 max_segments;
struct blk_mq_tag_set tag_set; struct blk_mq_tag_set tag_set;
u32 nr_poll_queues; u32 nr_poll_queues;
struct mutex lock; /* protects state and devs_list */ struct mutex lock; /* protects state and devs_list */

18
drivers/infiniband/ulp/rtrs/rtrs-clt.c
View File

@ -1357,7 +1357,6 @@ static void free_sess_reqs(struct rtrs_clt_sess *sess)
static int alloc_sess_reqs(struct rtrs_clt_sess *sess) static int alloc_sess_reqs(struct rtrs_clt_sess *sess)
{ {
struct rtrs_clt_io_req *req; struct rtrs_clt_io_req *req;
struct rtrs_clt *clt = sess->clt;
int i, err = -ENOMEM; int i, err = -ENOMEM;
sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs), sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs),
@ -1466,6 +1465,8 @@ static void query_fast_reg_mode(struct rtrs_clt_sess *sess)
sess->max_pages_per_mr = sess->max_pages_per_mr =
min3(sess->max_pages_per_mr, (u32)max_pages_per_mr, min3(sess->max_pages_per_mr, (u32)max_pages_per_mr,
ib_dev->attrs.max_fast_reg_page_list_len); ib_dev->attrs.max_fast_reg_page_list_len);
sess->clt->max_segments =
min(sess->max_pages_per_mr, sess->clt->max_segments);
} }
static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess, static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess,
@ -1503,9 +1504,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work);
static void rtrs_clt_close_work(struct work_struct *work); static void rtrs_clt_close_work(struct work_struct *work);
static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt,
const struct rtrs_addr *path, const struct rtrs_addr *path,
size_t con_num, u16 max_segments, size_t con_num, u32 nr_poll_queues)
u32 nr_poll_queues)
{ {
struct rtrs_clt_sess *sess; struct rtrs_clt_sess *sess;
int err = -ENOMEM; int err = -ENOMEM;
@ -2668,7 +2668,6 @@ static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num,
u16 port, size_t pdu_sz, void *priv, u16 port, size_t pdu_sz, void *priv,
void (*link_ev)(void *priv, void (*link_ev)(void *priv,
enum rtrs_clt_link_ev ev), enum rtrs_clt_link_ev ev),
unsigned int max_segments,
unsigned int reconnect_delay_sec, unsigned int reconnect_delay_sec,
unsigned int max_reconnect_attempts) unsigned int max_reconnect_attempts)
{ {
@ -2766,7 +2765,6 @@ static void free_clt(struct rtrs_clt *clt)
* @port: port to be used by the RTRS session * @port: port to be used by the RTRS session
* @pdu_sz: Size of extra payload which can be accessed after permit allocation. * @pdu_sz: Size of extra payload which can be accessed after permit allocation.
* @reconnect_delay_sec: time between reconnect tries * @reconnect_delay_sec: time between reconnect tries
* @max_segments: Max. number of segments per IO request
* @max_reconnect_attempts: Number of times to reconnect on error before giving * @max_reconnect_attempts: Number of times to reconnect on error before giving
* up, 0 for * disabled, -1 for forever * up, 0 for * disabled, -1 for forever
* @nr_poll_queues: number of polling mode connection using IB_POLL_DIRECT flag * @nr_poll_queues: number of polling mode connection using IB_POLL_DIRECT flag
@ -2781,7 +2779,6 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
const struct rtrs_addr *paths, const struct rtrs_addr *paths,
size_t paths_num, u16 port, size_t paths_num, u16 port,
size_t pdu_sz, u8 reconnect_delay_sec, size_t pdu_sz, u8 reconnect_delay_sec,
u16 max_segments,
s16 max_reconnect_attempts, u32 nr_poll_queues) s16 max_reconnect_attempts, u32 nr_poll_queues)
{ {
struct rtrs_clt_sess *sess, *tmp; struct rtrs_clt_sess *sess, *tmp;
@ -2790,7 +2787,7 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv, clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv,
ops->link_ev, ops->link_ev,
max_segments, reconnect_delay_sec, reconnect_delay_sec,
max_reconnect_attempts); max_reconnect_attempts);
if (IS_ERR(clt)) { if (IS_ERR(clt)) {
err = PTR_ERR(clt); err = PTR_ERR(clt);
@ -2800,7 +2797,7 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
struct rtrs_clt_sess *sess; struct rtrs_clt_sess *sess;
sess = alloc_sess(clt, &paths[i], nr_cpu_ids, sess = alloc_sess(clt, &paths[i], nr_cpu_ids,
max_segments, nr_poll_queues); nr_poll_queues);
if (IS_ERR(sess)) { if (IS_ERR(sess)) {
err = PTR_ERR(sess); err = PTR_ERR(sess);
goto close_all_sess; goto close_all_sess;
@ -3062,6 +3059,7 @@ int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr)
return -ECOMM; return -ECOMM;
attr->queue_depth = clt->queue_depth; attr->queue_depth = clt->queue_depth;
attr->max_segments = clt->max_segments;
/* Cap max_io_size to min of remote buffer size and the fr pages */ /* Cap max_io_size to min of remote buffer size and the fr pages */
attr->max_io_size = min_t(int, clt->max_io_size, attr->max_io_size = min_t(int, clt->max_io_size,
clt->max_segments * SZ_4K); clt->max_segments * SZ_4K);
@ -3076,7 +3074,7 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
struct rtrs_clt_sess *sess; struct rtrs_clt_sess *sess;
int err; int err;
sess = alloc_sess(clt, addr, nr_cpu_ids, clt->max_segments, 0); sess = alloc_sess(clt, addr, nr_cpu_ids, 0);
if (IS_ERR(sess)) if (IS_ERR(sess))
return PTR_ERR(sess); return PTR_ERR(sess);

2
drivers/infiniband/ulp/rtrs/rtrs.h
View File

@ -57,7 +57,6 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
const struct rtrs_addr *paths, const struct rtrs_addr *paths,
size_t path_cnt, u16 port, size_t path_cnt, u16 port,
size_t pdu_sz, u8 reconnect_delay_sec, size_t pdu_sz, u8 reconnect_delay_sec,
u16 max_segments,
s16 max_reconnect_attempts, u32 nr_poll_queues); s16 max_reconnect_attempts, u32 nr_poll_queues);
void rtrs_clt_close(struct rtrs_clt *sess); void rtrs_clt_close(struct rtrs_clt *sess);
@ -110,6 +109,7 @@ int rtrs_clt_rdma_cq_direct(struct rtrs_clt *clt, unsigned int index);
struct rtrs_attrs { struct rtrs_attrs {
u32 queue_depth; u32 queue_depth;
u32 max_io_size; u32 max_io_size;
u32 max_segments;
}; };
int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr); int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr);