leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'add-ethtool-support-for-completion-queue-event-size'

Browse Source 
Subbaraya Sundeep says:

====================
Add ethtool support for completion queue event size

After a packet is sent or received by NIC then NIC posts
a completion queue event which consists of transmission status
(like send success or error) and received status(like
pointers to packet fragments). These completion events may
also use a ring similar to rx and tx rings. This patchset
introduces cqe-size ethtool parameter to modify the size
of the completion queue event if NIC hardware has that capability.
A bigger completion queue event can have more receive buffer pointers
inturn NIC can transfer a bigger frame from wire as long as
hardware(MAC) receive frame size limit is not exceeded.

Patch 1 adds support setting/getting cqe-size via
ethtool -G and ethtool -g.

Patch 2 includes octeontx2 driver changes to use
completion queue event size set from ethtool -G.
====================

Link: https://lore.kernel.org/r/1645555153-4932-1-git-send-email-sbhatta@marvell.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2022-02-23 20:33:15 -08:00
commit e422eef268
10 changed files with 55 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Documentation/networking/ethtool-netlink.rst
View File

@ -861,6 +861,7 @@ Kernel response contents:
``ETHTOOL_A_RINGS_TX`` u32 size of TX ring
``ETHTOOL_A_RINGS_RX_BUF_LEN`` u32 size of buffers on the ring
``ETHTOOL_A_RINGS_TCP_DATA_SPLIT`` u8 TCP header / data split
``ETHTOOL_A_RINGS_CQE_SIZE`` u32 Size of TX/RX CQE
==================================== ====== ===========================
``ETHTOOL_A_RINGS_TCP_DATA_SPLIT`` indicates whether the device is usable with
@ -885,6 +886,7 @@ Request contents:
``ETHTOOL_A_RINGS_RX_JUMBO`` u32 size of RX jumbo ring
``ETHTOOL_A_RINGS_TX`` u32 size of TX ring
``ETHTOOL_A_RINGS_RX_BUF_LEN`` u32 size of buffers on the ring
``ETHTOOL_A_RINGS_CQE_SIZE`` u32 Size of TX/RX CQE
==================================== ====== ===========================
Kernel checks that requested ring sizes do not exceed limits reported by
@ -892,6 +894,15 @@ driver. Driver may impose additional constraints and may not suspport all
attributes.
``ETHTOOL_A_RINGS_CQE_SIZE`` specifies the completion queue event size.
Completion queue events(CQE) are the events posted by NIC to indicate the
completion status of a packet when the packet is sent(like send success or
error) or received(like pointers to packet fragments). The CQE size parameter
enables to modify the CQE size other than default size if NIC supports it.
A bigger CQE can have more receive buffer pointers inturn NIC can transfer
a bigger frame from wire. Based on the NIC hardware, the overall completion
queue size can be adjusted in the driver if CQE size is modified.
CHANNELS_GET
============

4
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
View File

@ -1048,7 +1048,7 @@ int otx2_config_nix(struct otx2_nic *pfvf)
struct nix_lf_alloc_rsp *rsp;
int err;
pfvf->qset.xqe_size = NIX_XQESZ_W16 ? 128 : 512;
pfvf->qset.xqe_size = pfvf->hw.xqe_size;
/* Get memory to put this msg */
nixlf = otx2_mbox_alloc_msg_nix_lf_alloc(&pfvf->mbox);
@ -1061,7 +1061,7 @@ int otx2_config_nix(struct otx2_nic *pfvf)
nixlf->cq_cnt = pfvf->qset.cq_cnt;
nixlf->rss_sz = MAX_RSS_INDIR_TBL_SIZE;
nixlf->rss_grps = MAX_RSS_GROUPS;
nixlf->xqe_sz = NIX_XQESZ_W16;
nixlf->xqe_sz = pfvf->hw.xqe_size == 128 ? NIX_XQESZ_W16 : NIX_XQESZ_W64;
/* We don't know absolute NPA LF idx attached.
* AF will replace 'RVU_DEFAULT_PF_FUNC' with
* NPA LF attached to this RVU PF/VF.

1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
View File

@ -181,6 +181,7 @@ struct otx2_hw {
#define OTX2_DEFAULT_RBUF_LEN 2048
u16 rbuf_len;
u32 xqe_size;
/* NPA */
u32 stack_pg_ptrs; /* No of ptrs per stack page */

17
drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
View File

@ -372,6 +372,7 @@ static void otx2_get_ringparam(struct net_device *netdev,
ring->tx_max_pending = Q_COUNT(Q_SIZE_MAX);
ring->tx_pending = qs->sqe_cnt ? qs->sqe_cnt : Q_COUNT(Q_SIZE_4K);
kernel_ring->rx_buf_len = pfvf->hw.rbuf_len;
kernel_ring->cqe_size = pfvf->hw.xqe_size;
}
static int otx2_set_ringparam(struct net_device *netdev,
@ -382,6 +383,7 @@ static int otx2_set_ringparam(struct net_device *netdev,
struct otx2_nic *pfvf = netdev_priv(netdev);
u32 rx_buf_len = kernel_ring->rx_buf_len;
u32 old_rx_buf_len = pfvf->hw.rbuf_len;
u32 xqe_size = kernel_ring->cqe_size;
bool if_up = netif_running(netdev);
struct otx2_qset *qs = &pfvf->qset;
u32 rx_count, tx_count;
@ -398,6 +400,12 @@ static int otx2_set_ringparam(struct net_device *netdev,
return -EINVAL;
}
if (xqe_size != 128 && xqe_size != 512) {
netdev_err(netdev,
"Completion event size must be 128 or 512");
return -EINVAL;
}
/* Permitted lengths are 16 64 256 1K 4K 16K 64K 256K 1M */
rx_count = ring->rx_pending;
/* On some silicon variants a skid or reserved CQEs are
@ -416,7 +424,7 @@ static int otx2_set_ringparam(struct net_device *netdev,
tx_count = Q_COUNT(Q_SIZE(tx_count, 3));
if (tx_count == qs->sqe_cnt && rx_count == qs->rqe_cnt &&
rx_buf_len == old_rx_buf_len)
rx_buf_len == old_rx_buf_len && xqe_size == pfvf->hw.xqe_size)
return 0;
if (if_up)
@ -427,6 +435,7 @@ static int otx2_set_ringparam(struct net_device *netdev,
qs->rqe_cnt = rx_count;
pfvf->hw.rbuf_len = rx_buf_len;
pfvf->hw.xqe_size = xqe_size;
if (if_up)
return netdev->netdev_ops->ndo_open(netdev);
@ -1222,7 +1231,8 @@ end:
static const struct ethtool_ops otx2_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
ETHTOOL_COALESCE_MAX_FRAMES,
.supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN,
.supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN |
ETHTOOL_RING_USE_CQE_SIZE,
.get_link = otx2_get_link,
.get_drvinfo = otx2_get_drvinfo,
.get_strings = otx2_get_strings,
@ -1342,7 +1352,8 @@ static int otx2vf_get_link_ksettings(struct net_device *netdev,
static const struct ethtool_ops otx2vf_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
ETHTOOL_COALESCE_MAX_FRAMES,
.supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN,
.supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN |
ETHTOOL_RING_USE_CQE_SIZE,
.get_link = otx2_get_link,
.get_drvinfo = otx2vf_get_drvinfo,
.get_strings = otx2vf_get_strings,

2
drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
View File

@ -2585,6 +2585,8 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
hw->tot_tx_queues = qcount;
hw->max_queues = qcount;
hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN;
/* Use CQE of 128 byte descriptor size by default */
hw->xqe_size = 128;
num_vec = pci_msix_vec_count(pdev);
hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,

2
drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
View File

@ -572,6 +572,8 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
hw->max_queues = qcount;
hw->tot_tx_queues = qcount;
hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN;
/* Use CQE of 128 byte descriptor size by default */
hw->xqe_size = 128;
hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,
GFP_KERNEL);

4
include/linux/ethtool.h
View File

@ -71,18 +71,22 @@ enum {
* struct kernel_ethtool_ringparam - RX/TX ring configuration
* @rx_buf_len: Current length of buffers on the rx ring.
* @tcp_data_split: Scatter packet headers and data to separate buffers
* @cqe_size: Size of TX/RX completion queue event
*/
struct kernel_ethtool_ringparam {
u32 rx_buf_len;
u8 tcp_data_split;
u32 cqe_size;
};
/**
* enum ethtool_supported_ring_param - indicator caps for setting ring params
* @ETHTOOL_RING_USE_RX_BUF_LEN: capture for setting rx_buf_len
* @ETHTOOL_RING_USE_CQE_SIZE: capture for setting cqe_size
*/
enum ethtool_supported_ring_param {
ETHTOOL_RING_USE_RX_BUF_LEN = BIT(0),
ETHTOOL_RING_USE_CQE_SIZE = BIT(1),
};
#define __ETH_RSS_HASH_BIT(bit) ((u32)1 << (bit))

1
include/uapi/linux/ethtool_netlink.h
View File

@ -337,6 +337,7 @@ enum {
ETHTOOL_A_RINGS_TX, /* u32 */
ETHTOOL_A_RINGS_RX_BUF_LEN, /* u32 */
ETHTOOL_A_RINGS_TCP_DATA_SPLIT, /* u8 */
ETHTOOL_A_RINGS_CQE_SIZE, /* u32 */
/* add new constants above here */
__ETHTOOL_A_RINGS_CNT,

2
net/ethtool/netlink.h
View File

@ -363,7 +363,7 @@ extern const struct nla_policy ethnl_features_set_policy[ETHTOOL_A_FEATURES_WANT
extern const struct nla_policy ethnl_privflags_get_policy[ETHTOOL_A_PRIVFLAGS_HEADER + 1];
extern const struct nla_policy ethnl_privflags_set_policy[ETHTOOL_A_PRIVFLAGS_FLAGS + 1];
extern const struct nla_policy ethnl_rings_get_policy[ETHTOOL_A_RINGS_HEADER + 1];
extern const struct nla_policy ethnl_rings_set_policy[ETHTOOL_A_RINGS_RX_BUF_LEN + 1];
extern const struct nla_policy ethnl_rings_set_policy[ETHTOOL_A_RINGS_CQE_SIZE + 1];
extern const struct nla_policy ethnl_channels_get_policy[ETHTOOL_A_CHANNELS_HEADER + 1];
extern const struct nla_policy ethnl_channels_set_policy[ETHTOOL_A_CHANNELS_COMBINED_COUNT + 1];
extern const struct nla_policy ethnl_coalesce_get_policy[ETHTOOL_A_COALESCE_HEADER + 1];

19
net/ethtool/rings.c
View File

@ -54,7 +54,8 @@ static int rings_reply_size(const struct ethnl_req_info *req_base,
nla_total_size(sizeof(u32)) + /* _RINGS_RX_JUMBO */
nla_total_size(sizeof(u32)) + /* _RINGS_TX */
nla_total_size(sizeof(u32)) + /* _RINGS_RX_BUF_LEN */
nla_total_size(sizeof(u8)); /* _RINGS_TCP_DATA_SPLIT */
nla_total_size(sizeof(u8)) + /* _RINGS_TCP_DATA_SPLIT */
nla_total_size(sizeof(u32)); /* _RINGS_CQE_SIZE */
}
static int rings_fill_reply(struct sk_buff *skb,
@ -91,7 +92,9 @@ static int rings_fill_reply(struct sk_buff *skb,
(nla_put_u32(skb, ETHTOOL_A_RINGS_RX_BUF_LEN, kr->rx_buf_len))) ||
(kr->tcp_data_split &&
(nla_put_u8(skb, ETHTOOL_A_RINGS_TCP_DATA_SPLIT,
kr->tcp_data_split))))
kr->tcp_data_split))) ||
(kr->cqe_size &&
(nla_put_u32(skb, ETHTOOL_A_RINGS_CQE_SIZE, kr->cqe_size))))
return -EMSGSIZE;
return 0;
@ -119,6 +122,7 @@ const struct nla_policy ethnl_rings_set_policy[] = {
[ETHTOOL_A_RINGS_RX_JUMBO] = { .type = NLA_U32 },
[ETHTOOL_A_RINGS_TX] = { .type = NLA_U32 },
[ETHTOOL_A_RINGS_RX_BUF_LEN] = NLA_POLICY_MIN(NLA_U32, 1),
[ETHTOOL_A_RINGS_CQE_SIZE] = NLA_POLICY_MIN(NLA_U32, 1),
};
int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info)
@ -159,6 +163,8 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info)
ethnl_update_u32(&ringparam.tx_pending, tb[ETHTOOL_A_RINGS_TX], &mod);
ethnl_update_u32(&kernel_ringparam.rx_buf_len,
tb[ETHTOOL_A_RINGS_RX_BUF_LEN], &mod);
ethnl_update_u32(&kernel_ringparam.cqe_size,
tb[ETHTOOL_A_RINGS_CQE_SIZE], &mod);
ret = 0;
if (!mod)
goto out_ops;
@ -190,6 +196,15 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info)
goto out_ops;
}
if (kernel_ringparam.cqe_size &&
!(ops->supported_ring_params & ETHTOOL_RING_USE_CQE_SIZE)) {
ret = -EOPNOTSUPP;
NL_SET_ERR_MSG_ATTR(info->extack,
tb[ETHTOOL_A_RINGS_CQE_SIZE],
"setting cqe size not supported");
goto out_ops;
}
ret = dev->ethtool_ops->set_ringparam(dev, &ringparam,
&kernel_ringparam, info->extack);
if (ret < 0)