leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

net/mlx5e: HTB, move htb functions to a new file

Browse Source 
Move htb related functions and data to a separated file for better
encapsulation.

Signed-off-by: Moshe Tal <moshet@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
This commit is contained in:
Moshe Tal 2022-05-02 11:31:26 +03:00 committed by Saeed Mahameed
parent 3685eed56f
commit 462b005999
7 changed files with 832 additions and 746 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/net/ethernet/mellanox/mlx5/core/Makefile
View File

@ -28,7 +28,8 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \
en_selftest.o en/port.o en/monitor_stats.o en/health.o \
en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \
en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \
en/qos.o en/trap.o en/fs_tt_redirect.o en/selq.o lib/crypto.o
en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \
lib/crypto.o
#
# Netdev extra

722
drivers/net/ethernet/mellanox/mlx5/core/en/htb.c Normal file
View File

@ -0,0 +1,722 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
#include <net/pkt_cls.h>
#include "htb.h"
#include "en.h"
#include "../qos.h"
struct mlx5e_qos_node {
struct hlist_node hnode;
struct mlx5e_qos_node *parent;
u64 rate;
u32 bw_share;
u32 max_average_bw;
u32 hw_id;
u32 classid; /* 16-bit, except root. */
u16 qid;
};
struct mlx5e_htb {
DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES));
DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES);
struct mlx5_core_dev *mdev;
struct net_device *netdev;
struct mlx5e_priv *priv;
struct mlx5e_selq *selq;
};
#define MLX5E_QOS_QID_INNER 0xffff
#define MLX5E_HTB_CLASSID_ROOT 0xffffffff
/* Software representation of the QoS tree */
int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data)
{
struct mlx5e_qos_node *node = NULL;
int bkt, err;
hash_for_each(htb->qos_tc2node, bkt, node, hnode) {
if (node->qid == MLX5E_QOS_QID_INNER)
continue;
err = callback(data, node->qid, node->hw_id);
if (err)
return err;
}
return 0;
}
int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb)
{
int last;
last = find_last_bit(htb->qos_used_qids, mlx5e_qos_max_leaf_nodes(htb->mdev));
return last == mlx5e_qos_max_leaf_nodes(htb->mdev) ? 0 : last + 1;
}
static int mlx5e_htb_find_unused_qos_qid(struct mlx5e_htb *htb)
{
int size = mlx5e_qos_max_leaf_nodes(htb->mdev);
struct mlx5e_priv *priv = htb->priv;
int res;
WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__);
res = find_first_zero_bit(htb->qos_used_qids, size);
return res == size ? -ENOSPC : res;
}
static struct mlx5e_qos_node *
mlx5e_htb_node_create_leaf(struct mlx5e_htb *htb, u16 classid, u16 qid,
struct mlx5e_qos_node *parent)
{
struct mlx5e_qos_node *node;
node = kzalloc(sizeof(*node), GFP_KERNEL);
if (!node)
return ERR_PTR(-ENOMEM);
node->parent = parent;
node->qid = qid;
__set_bit(qid, htb->qos_used_qids);
node->classid = classid;
hash_add_rcu(htb->qos_tc2node, &node->hnode, classid);
mlx5e_update_tx_netdev_queues(htb->priv);
return node;
}
static struct mlx5e_qos_node *mlx5e_htb_node_create_root(struct mlx5e_htb *htb)
{
struct mlx5e_qos_node *node;
node = kzalloc(sizeof(*node), GFP_KERNEL);
if (!node)
return ERR_PTR(-ENOMEM);
node->qid = MLX5E_QOS_QID_INNER;
node->classid = MLX5E_HTB_CLASSID_ROOT;
hash_add_rcu(htb->qos_tc2node, &node->hnode, node->classid);
return node;
}
static struct mlx5e_qos_node *mlx5e_htb_node_find(struct mlx5e_htb *htb, u32 classid)
{
struct mlx5e_qos_node *node = NULL;
hash_for_each_possible(htb->qos_tc2node, node, hnode, classid) {
if (node->classid == classid)
break;
}
return node;
}
static struct mlx5e_qos_node *mlx5e_htb_node_find_rcu(struct mlx5e_htb *htb, u32 classid)
{
struct mlx5e_qos_node *node = NULL;
hash_for_each_possible_rcu(htb->qos_tc2node, node, hnode, classid) {
if (node->classid == classid)
break;
}
return node;
}
static void mlx5e_htb_node_delete(struct mlx5e_htb *htb, struct mlx5e_qos_node *node)
{
hash_del_rcu(&node->hnode);
if (node->qid != MLX5E_QOS_QID_INNER) {
__clear_bit(node->qid, htb->qos_used_qids);
mlx5e_update_tx_netdev_queues(htb->priv);
}
/* Make sure this qid is no longer selected by mlx5e_select_queue, so
* that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue.
*/
synchronize_net();
kfree(node);
}
/* TX datapath API */
int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid)
{
struct mlx5e_qos_node *node;
u16 qid;
int res;
rcu_read_lock();
node = mlx5e_htb_node_find_rcu(htb, classid);
if (!node) {
res = -ENOENT;
goto out;
}
qid = READ_ONCE(node->qid);
if (qid == MLX5E_QOS_QID_INNER) {
res = -EINVAL;
goto out;
}
res = mlx5e_qid_from_qos(&htb->priv->channels, qid);
out:
rcu_read_unlock();
return res;
}
/* HTB TC handlers */
static int
mlx5e_htb_root_add(struct mlx5e_htb *htb, u16 htb_maj_id, u16 htb_defcls,
struct netlink_ext_ack *extack)
{
struct mlx5e_priv *priv = htb->priv;
struct mlx5e_qos_node *root;
bool opened;
int err;
qos_dbg(htb->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls);
mlx5e_selq_prepare_htb(htb->selq, htb_maj_id, htb_defcls);
opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (opened) {
err = mlx5e_qos_alloc_queues(priv, &priv->channels);
if (err)
goto err_cancel_selq;
}
root = mlx5e_htb_node_create_root(htb);
if (IS_ERR(root)) {
err = PTR_ERR(root);
goto err_free_queues;
}
err = mlx5_qos_create_root_node(htb->mdev, &root->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware.");
goto err_sw_node_delete;
}
mlx5e_selq_apply(htb->selq);
return 0;
err_sw_node_delete:
mlx5e_htb_node_delete(htb, root);
err_free_queues:
if (opened)
mlx5e_qos_close_all_queues(&priv->channels);
err_cancel_selq:
mlx5e_selq_cancel(htb->selq);
return err;
}
static int mlx5e_htb_root_del(struct mlx5e_htb *htb)
{
struct mlx5e_priv *priv = htb->priv;
struct mlx5e_qos_node *root;
int err;
qos_dbg(htb->mdev, "TC_HTB_DESTROY\n");
/* Wait until real_num_tx_queues is updated for mlx5e_select_queue,
* so that we can safely switch to its non-HTB non-PTP fastpath.
*/
synchronize_net();
mlx5e_selq_prepare_htb(htb->selq, 0, 0);
mlx5e_selq_apply(htb->selq);
root = mlx5e_htb_node_find(htb, MLX5E_HTB_CLASSID_ROOT);
if (!root) {
qos_err(htb->mdev, "Failed to find the root node in the QoS tree\n");
return -ENOENT;
}
err = mlx5_qos_destroy_node(htb->mdev, root->hw_id);
if (err)
qos_err(htb->mdev, "Failed to destroy root node %u, err = %d\n",
root->hw_id, err);
mlx5e_htb_node_delete(htb, root);
mlx5e_qos_deactivate_all_queues(&priv->channels);
mlx5e_qos_close_all_queues(&priv->channels);
return err;
}
static int mlx5e_htb_convert_rate(struct mlx5e_htb *htb, u64 rate,
struct mlx5e_qos_node *parent, u32 *bw_share)
{
u64 share = 0;
while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw)
parent = parent->parent;
if (parent->max_average_bw)
share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT),
parent->max_average_bw);
else
share = 101;
*bw_share = share == 0 ? 1 : share > 100 ? 0 : share;
qos_dbg(htb->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n",
rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share);
return 0;
}
static void mlx5e_htb_convert_ceil(struct mlx5e_htb *htb, u64 ceil, u32 *max_average_bw)
{
/* Hardware treats 0 as "unlimited", set at least 1. */
*max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1);
qos_dbg(htb->mdev, "Convert: ceil %llu -> max_average_bw %u\n",
ceil, *max_average_bw);
}
int
mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid,
u32 parent_classid, u64 rate, u64 ceil,
struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *node, *parent;
struct mlx5e_priv *priv = htb->priv;
int qid;
int err;
qos_dbg(htb->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n",
classid, parent_classid, rate, ceil);
qid = mlx5e_htb_find_unused_qos_qid(htb);
if (qid < 0) {
NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached.");
return qid;
}
parent = mlx5e_htb_node_find(htb, parent_classid);
if (!parent)
return -EINVAL;
node = mlx5e_htb_node_create_leaf(htb, classid, qid, parent);
if (IS_ERR(node))
return PTR_ERR(node);
node->rate = rate;
mlx5e_htb_convert_rate(htb, rate, node->parent, &node->bw_share);
mlx5e_htb_convert_ceil(htb, ceil, &node->max_average_bw);
err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->hw_id,
node->bw_share, node->max_average_bw,
&node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node.");
qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n",
classid, err);
mlx5e_htb_node_delete(htb, node);
return err;
}
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n",
classid, err);
} else {
mlx5e_activate_qos_sq(priv, node->qid, node->hw_id);
}
}
return mlx5e_qid_from_qos(&priv->channels, node->qid);
}
int
mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid,
u64 rate, u64 ceil, struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *node, *child;
struct mlx5e_priv *priv = htb->priv;
int err, tmp_err;
u32 new_hw_id;
u16 qid;
qos_dbg(htb->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n",
classid, child_classid, rate, ceil);
node = mlx5e_htb_node_find(htb, classid);
if (!node)
return -ENOENT;
err = mlx5_qos_create_inner_node(htb->mdev, node->parent->hw_id,
node->bw_share, node->max_average_bw,
&new_hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node.");
qos_err(htb->mdev, "Failed to create an inner node (class %04x), err = %d\n",
classid, err);
return err;
}
/* Intentionally reuse the qid for the upcoming first child. */
child = mlx5e_htb_node_create_leaf(htb, child_classid, node->qid, node);
if (IS_ERR(child)) {
err = PTR_ERR(child);
goto err_destroy_hw_node;
}
child->rate = rate;
mlx5e_htb_convert_rate(htb, rate, node, &child->bw_share);
mlx5e_htb_convert_ceil(htb, ceil, &child->max_average_bw);
err = mlx5_qos_create_leaf_node(htb->mdev, new_hw_id, child->bw_share,
child->max_average_bw, &child->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node.");
qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n",
classid, err);
goto err_delete_sw_node;
}
/* No fail point. */
qid = node->qid;
/* Pairs with mlx5e_htb_get_txq_by_classid. */
WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER);
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
mlx5e_deactivate_qos_sq(priv, qid);
mlx5e_close_qos_sq(priv, qid);
}
err = mlx5_qos_destroy_node(htb->mdev, node->hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, classid, err);
node->hw_id = new_hw_id;
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, child->qid, child->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n",
classid, err);
} else {
mlx5e_activate_qos_sq(priv, child->qid, child->hw_id);
}
}
return 0;
err_delete_sw_node:
child->qid = MLX5E_QOS_QID_INNER;
mlx5e_htb_node_delete(htb, child);
err_destroy_hw_node:
tmp_err = mlx5_qos_destroy_node(htb->mdev, new_hw_id);
if (tmp_err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n",
new_hw_id, classid, tmp_err);
return err;
}
static struct mlx5e_qos_node *mlx5e_htb_node_find_by_qid(struct mlx5e_htb *htb, u16 qid)
{
struct mlx5e_qos_node *node = NULL;
int bkt;
hash_for_each(htb->qos_tc2node, bkt, node, hnode)
if (node->qid == qid)
break;
return node;
}
int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid,
struct netlink_ext_ack *extack)
{
struct mlx5e_priv *priv = htb->priv;
struct mlx5e_qos_node *node;
struct netdev_queue *txq;
u16 qid, moved_qid;
bool opened;
int err;
qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid);
node = mlx5e_htb_node_find(htb, *classid);
if (!node)
return -ENOENT;
/* Store qid for reuse. */
qid = node->qid;
opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (opened) {
txq = netdev_get_tx_queue(htb->netdev,
mlx5e_qid_from_qos(&priv->channels, qid));
mlx5e_deactivate_qos_sq(priv, qid);
mlx5e_close_qos_sq(priv, qid);
}
err = mlx5_qos_destroy_node(htb->mdev, node->hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, *classid, err);
mlx5e_htb_node_delete(htb, node);
moved_qid = mlx5e_htb_cur_leaf_nodes(htb);
if (moved_qid == 0) {
/* The last QoS SQ was just destroyed. */
if (opened)
mlx5e_reactivate_qos_sq(priv, qid, txq);
return 0;
}
moved_qid--;
if (moved_qid < qid) {
/* The highest QoS SQ was just destroyed. */
WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u",
qid, moved_qid);
if (opened)
mlx5e_reactivate_qos_sq(priv, qid, txq);
return 0;
}
WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid);
qos_dbg(htb->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid);
node = mlx5e_htb_node_find_by_qid(htb, moved_qid);
WARN(!node, "Could not find a node with qid %u to move to queue %u",
moved_qid, qid);
/* Stop traffic to the old queue. */
WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER);
__clear_bit(moved_qid, priv->htb->qos_used_qids);
if (opened) {
txq = netdev_get_tx_queue(htb->netdev,
mlx5e_qid_from_qos(&priv->channels, moved_qid));
mlx5e_deactivate_qos_sq(priv, moved_qid);
mlx5e_close_qos_sq(priv, moved_qid);
}
/* Prevent packets from the old class from getting into the new one. */
mlx5e_reset_qdisc(htb->netdev, moved_qid);
__set_bit(qid, htb->qos_used_qids);
WRITE_ONCE(node->qid, qid);
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n",
node->classid, moved_qid, qid, err);
} else {
mlx5e_activate_qos_sq(priv, node->qid, node->hw_id);
}
}
mlx5e_update_tx_netdev_queues(priv);
if (opened)
mlx5e_reactivate_qos_sq(priv, moved_qid, txq);
*classid = node->classid;
return 0;
}
int
mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force,
struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *node, *parent;
struct mlx5e_priv *priv = htb->priv;
u32 old_hw_id, new_hw_id;
int err, saved_err = 0;
u16 qid;
qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n",
force ? "_FORCE" : "", classid);
node = mlx5e_htb_node_find(htb, classid);
if (!node)
return -ENOENT;
err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->parent->hw_id,
node->parent->bw_share,
node->parent->max_average_bw,
&new_hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node.");
qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n",
classid, err);
if (!force)
return err;
saved_err = err;
}
/* Store qid for reuse and prevent clearing the bit. */
qid = node->qid;
/* Pairs with mlx5e_htb_get_txq_by_classid. */
WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER);
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
mlx5e_deactivate_qos_sq(priv, qid);
mlx5e_close_qos_sq(priv, qid);
}
/* Prevent packets from the old class from getting into the new one. */
mlx5e_reset_qdisc(htb->netdev, qid);
err = mlx5_qos_destroy_node(htb->mdev, node->hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, classid, err);
parent = node->parent;
mlx5e_htb_node_delete(htb, node);
node = parent;
WRITE_ONCE(node->qid, qid);
/* Early return on error in force mode. Parent will still be an inner
* node to be deleted by a following delete operation.
*/
if (saved_err)
return saved_err;
old_hw_id = node->hw_id;
node->hw_id = new_hw_id;
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n",
classid, err);
} else {
mlx5e_activate_qos_sq(priv, node->qid, node->hw_id);
}
}
err = mlx5_qos_destroy_node(htb->mdev, old_hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, classid, err);
return 0;
}
static int
mlx5e_htb_update_children(struct mlx5e_htb *htb, struct mlx5e_qos_node *node,
struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *child;
int err = 0;
int bkt;
hash_for_each(htb->qos_tc2node, bkt, child, hnode) {
u32 old_bw_share = child->bw_share;
int err_one;
if (child->parent != node)
continue;
mlx5e_htb_convert_rate(htb, child->rate, node, &child->bw_share);
if (child->bw_share == old_bw_share)
continue;
err_one = mlx5_qos_update_node(htb->mdev, child->hw_id, child->bw_share,
child->max_average_bw, child->hw_id);
if (!err && err_one) {
err = err_one;
NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node.");
qos_err(htb->mdev, "Failed to modify a child node (class %04x), err = %d\n",
node->classid, err);
}
}
return err;
}
int
mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil,
struct netlink_ext_ack *extack)
{
u32 bw_share, max_average_bw;
struct mlx5e_qos_node *node;
bool ceil_changed = false;
int err;
qos_dbg(htb->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n",
classid, rate, ceil);
node = mlx5e_htb_node_find(htb, classid);
if (!node)
return -ENOENT;
node->rate = rate;
mlx5e_htb_convert_rate(htb, rate, node->parent, &bw_share);
mlx5e_htb_convert_ceil(htb, ceil, &max_average_bw);
err = mlx5_qos_update_node(htb->mdev, node->parent->hw_id, bw_share,
max_average_bw, node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node.");
qos_err(htb->mdev, "Failed to modify a node (class %04x), err = %d\n",
classid, err);
return err;
}
if (max_average_bw != node->max_average_bw)
ceil_changed = true;
node->bw_share = bw_share;
node->max_average_bw = max_average_bw;
if (ceil_changed)
err = mlx5e_htb_update_children(htb, node, extack);
return err;
}
struct mlx5e_htb *mlx5e_htb_alloc(void)
{
return kvzalloc(sizeof(struct mlx5e_htb), GFP_KERNEL);
}
void mlx5e_htb_free(struct mlx5e_htb *htb)
{
kvfree(htb);
}
int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt,
struct net_device *netdev, struct mlx5_core_dev *mdev,
struct mlx5e_selq *selq, struct mlx5e_priv *priv)
{
htb->mdev = mdev;
htb->netdev = netdev;
htb->selq = selq;
htb->priv = priv;
hash_init(htb->qos_tc2node);
return mlx5e_htb_root_add(htb, htb_qopt->parent_classid, htb_qopt->classid,
htb_qopt->extack);
}
void mlx5e_htb_cleanup(struct mlx5e_htb *htb)
{
mlx5e_htb_root_del(htb);
}

46
drivers/net/ethernet/mellanox/mlx5/core/en/htb.h Normal file
View File

@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
#ifndef __MLX5E_EN_HTB_H_
#define __MLX5E_EN_HTB_H_
#include "qos.h"
#define MLX5E_QOS_MAX_LEAF_NODES 256
struct mlx5e_selq;
struct mlx5e_htb;
typedef int (*mlx5e_fp_htb_enumerate)(void *data, u16 qid, u32 hw_id);
int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data);
int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb);
/* TX datapath API */
int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid);
/* HTB TC handlers */
int
mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid,
u32 parent_classid, u64 rate, u64 ceil,
struct netlink_ext_ack *extack);
int
mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid,
u64 rate, u64 ceil, struct netlink_ext_ack *extack);
int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid,
struct netlink_ext_ack *extack);
int
mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force,
struct netlink_ext_ack *extack);
int
mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil,
struct netlink_ext_ack *extack);
struct mlx5e_htb *mlx5e_htb_alloc(void);
void mlx5e_htb_free(struct mlx5e_htb *htb);
int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt,
struct net_device *netdev, struct mlx5_core_dev *mdev,
struct mlx5e_selq *selq, struct mlx5e_priv *priv);
void mlx5e_htb_cleanup(struct mlx5e_htb *htb);
#endif

785
drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
View File

@ -6,16 +6,11 @@
#include "en.h"
#include "params.h"
#include "../qos.h"
#include "en/htb.h"
#define BYTES_IN_MBIT 125000
struct mlx5e_htb {
DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES));
DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES);
struct mlx5_core_dev *mdev;
struct net_device *netdev;
struct qos_sq_callback_params {
struct mlx5e_priv *priv;
struct mlx5e_selq *selq;
struct mlx5e_channels *chs;
};
int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes)
@ -38,121 +33,9 @@ int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev)
return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev));
}
int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb)
{
int last;
last = find_last_bit(htb->qos_used_qids, mlx5e_qos_max_leaf_nodes(htb->mdev));
return last == mlx5e_qos_max_leaf_nodes(htb->mdev) ? 0 : last + 1;
}
/* Software representation of the QoS tree (internal to this file) */
static int mlx5e_find_unused_qos_qid(struct mlx5e_htb *htb)
{
int size = mlx5e_qos_max_leaf_nodes(htb->mdev);
struct mlx5e_priv *priv = htb->priv;
int res;
WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__);
res = find_first_zero_bit(htb->qos_used_qids, size);
return res == size ? -ENOSPC : res;
}
struct mlx5e_qos_node {
struct hlist_node hnode;
struct mlx5e_qos_node *parent;
u64 rate;
u32 bw_share;
u32 max_average_bw;
u32 hw_id;
u32 classid; /* 16-bit, except root. */
u16 qid;
};
#define MLX5E_QOS_QID_INNER 0xffff
#define MLX5E_HTB_CLASSID_ROOT 0xffffffff
static struct mlx5e_qos_node *
mlx5e_htb_node_create_leaf(struct mlx5e_htb *htb, u16 classid, u16 qid,
struct mlx5e_qos_node *parent)
{
struct mlx5e_qos_node *node;
node = kzalloc(sizeof(*node), GFP_KERNEL);
if (!node)
return ERR_PTR(-ENOMEM);
node->parent = parent;
node->qid = qid;
__set_bit(qid, htb->qos_used_qids);
node->classid = classid;
hash_add_rcu(htb->qos_tc2node, &node->hnode, classid);
mlx5e_update_tx_netdev_queues(htb->priv);
return node;
}
static struct mlx5e_qos_node *mlx5e_htb_node_create_root(struct mlx5e_htb *htb)
{
struct mlx5e_qos_node *node;
node = kzalloc(sizeof(*node), GFP_KERNEL);
if (!node)
return ERR_PTR(-ENOMEM);
node->qid = MLX5E_QOS_QID_INNER;
node->classid = MLX5E_HTB_CLASSID_ROOT;
hash_add_rcu(htb->qos_tc2node, &node->hnode, node->classid);
return node;
}
static struct mlx5e_qos_node *mlx5e_htb_node_find(struct mlx5e_htb *htb, u32 classid)
{
struct mlx5e_qos_node *node = NULL;
hash_for_each_possible(htb->qos_tc2node, node, hnode, classid) {
if (node->classid == classid)
break;
}
return node;
}
static struct mlx5e_qos_node *mlx5e_htb_node_find_rcu(struct mlx5e_htb *htb, u32 classid)
{
struct mlx5e_qos_node *node = NULL;
hash_for_each_possible_rcu(htb->qos_tc2node, node, hnode, classid) {
if (node->classid == classid)
break;
}
return node;
}
static void mlx5e_htb_node_delete(struct mlx5e_htb *htb, struct mlx5e_qos_node *node)
{
hash_del_rcu(&node->hnode);
if (node->qid != MLX5E_QOS_QID_INNER) {
__clear_bit(node->qid, htb->qos_used_qids);
mlx5e_update_tx_netdev_queues(htb->priv);
}
/* Make sure this qid is no longer selected by mlx5e_select_queue, so
* that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue.
*/
synchronize_net();
kfree(node);
}
/* TX datapath API */
static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid)
u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid)
{
/* These channel params are safe to access from the datapath, because:
* 1. This function is called only after checking selq->htb_maj_id != 0,
@ -166,31 +49,6 @@ static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid)
return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid;
}
int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid)
{
struct mlx5e_qos_node *node;
u16 qid;
int res;
rcu_read_lock();
node = mlx5e_htb_node_find_rcu(htb, classid);
if (!node) {
res = -ENOENT;
goto out;
}
qid = READ_ONCE(node->qid);
if (qid == MLX5E_QOS_QID_INNER) {
res = -EINVAL;
goto out;
}
res = mlx5e_qid_from_qos(&htb->priv->channels, qid);
out:
rcu_read_unlock();
return res;
}
/* SQ lifecycle */
static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid)
@ -208,8 +66,8 @@ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid)
return mlx5e_state_dereference(priv, qos_sqs[qid]);
}
static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs,
struct mlx5e_qos_node *node)
int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs,
u16 node_qid, u32 hw_id)
{
struct mlx5e_create_cq_param ccp = {};
struct mlx5e_txqsq __rcu **qos_sqs;
@ -222,10 +80,10 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs
params = &chs->params;
txq_ix = mlx5e_qid_from_qos(chs, node->qid);
txq_ix = mlx5e_qid_from_qos(chs, node_qid);
WARN_ON(node->qid > priv->htb_max_qos_sqs);
if (node->qid == priv->htb_max_qos_sqs) {
WARN_ON(node_qid > priv->htb_max_qos_sqs);
if (node_qid == priv->htb_max_qos_sqs) {
struct mlx5e_sq_stats *stats, **stats_list = NULL;
if (priv->htb_max_qos_sqs == 0) {
@ -242,15 +100,15 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs
}
if (stats_list)
WRITE_ONCE(priv->htb_qos_sq_stats, stats_list);
WRITE_ONCE(priv->htb_qos_sq_stats[node->qid], stats);
WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats);
/* Order htb_max_qos_sqs increment after writing the array pointer.
* Pairs with smp_load_acquire in en_stats.c.
*/
smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1);
}
ix = node->qid % params->num_channels;
qid = node->qid / params->num_channels;
ix = node_qid % params->num_channels;
qid = node_qid / params->num_channels;
c = chs->c[ix];
qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
@ -269,8 +127,8 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs
if (err)
goto err_free_sq;
err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params,
&param_sq, sq, 0, node->hw_id,
priv->htb_qos_sq_stats[node->qid]);
&param_sq, sq, 0, hw_id,
priv->htb_qos_sq_stats[node_qid]);
if (err)
goto err_close_cq;
@ -285,14 +143,22 @@ err_free_sq:
return err;
}
static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node *node)
static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id)
{
struct qos_sq_callback_params *cb_params = data;
return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id);
}
int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id)
{
struct mlx5e_priv *priv = data;
struct mlx5e_txqsq *sq;
u16 qid;
sq = mlx5e_get_qos_sq(priv, node->qid);
sq = mlx5e_get_qos_sq(priv, node_qid);
qid = mlx5e_qid_from_qos(&priv->channels, node->qid);
qid = mlx5e_qid_from_qos(&priv->channels, node_qid);
/* If it's a new queue, it will be marked as started at this point.
* Stop it before updating txq2sq.
@ -307,11 +173,13 @@ static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node
*/
smp_wmb();
qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node->qid);
qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid);
mlx5e_activate_txqsq(sq);
return 0;
}
static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid)
void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid)
{
struct mlx5e_txqsq *sq;
@ -331,7 +199,7 @@ static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid)
smp_wmb();
}
static void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid)
void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid)
{
struct mlx5e_txqsq __rcu **qos_sqs;
struct mlx5e_params *params;
@ -381,7 +249,7 @@ void mlx5e_qos_close_queues(struct mlx5e_channel *c)
kvfree(qos_sqs);
}
static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
{
int i;
@ -389,7 +257,7 @@ static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
mlx5e_qos_close_queues(chs->c[i]);
}
static int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
{
u16 qos_sqs_size;
int i;
@ -425,21 +293,20 @@ err_free:
int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
{
struct mlx5e_qos_node *node = NULL;
int bkt, err;
struct qos_sq_callback_params callback_params;
int err;
err = mlx5e_qos_alloc_queues(priv, chs);
if (err)
return err;
hash_for_each(priv->htb->qos_tc2node, bkt, node, hnode) {
if (node->qid == MLX5E_QOS_QID_INNER)
continue;
err = mlx5e_open_qos_sq(priv, chs, node);
if (err) {
mlx5e_qos_close_all_queues(chs);
return err;
}
callback_params.priv = priv;
callback_params.chs = chs;
err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params);
if (err) {
mlx5e_qos_close_all_queues(chs);
return err;
}
return 0;
@ -447,14 +314,7 @@ int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
void mlx5e_qos_activate_queues(struct mlx5e_priv *priv)
{
struct mlx5e_qos_node *node = NULL;
int bkt;
hash_for_each(priv->htb->qos_tc2node, bkt, node, hnode) {
if (node->qid == MLX5E_QOS_QID_INNER)
continue;
mlx5e_activate_qos_sq(priv, node);
}
mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv);
}
void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
@ -483,7 +343,7 @@ void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
}
}
static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
{
int i;
@ -491,285 +351,14 @@ static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
mlx5e_qos_deactivate_queues(chs->c[i]);
}
/* HTB TC handlers */
static int
mlx5e_htb_root_add(struct mlx5e_htb *htb, u16 htb_maj_id, u16 htb_defcls,
struct netlink_ext_ack *extack)
{
struct mlx5e_priv *priv = htb->priv;
struct mlx5e_qos_node *root;
bool opened;
int err;
qos_dbg(htb->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls);
mlx5e_selq_prepare_htb(htb->selq, htb_maj_id, htb_defcls);
opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (opened) {
err = mlx5e_qos_alloc_queues(priv, &priv->channels);
if (err)
goto err_cancel_selq;
}
root = mlx5e_htb_node_create_root(htb);
if (IS_ERR(root)) {
err = PTR_ERR(root);
goto err_free_queues;
}
err = mlx5_qos_create_root_node(htb->mdev, &root->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware.");
goto err_sw_node_delete;
}
mlx5e_selq_apply(htb->selq);
return 0;
err_sw_node_delete:
mlx5e_htb_node_delete(htb, root);
err_free_queues:
if (opened)
mlx5e_qos_close_all_queues(&priv->channels);
err_cancel_selq:
mlx5e_selq_cancel(htb->selq);
return err;
}
static int mlx5e_htb_root_del(struct mlx5e_htb *htb)
{
struct mlx5e_priv *priv = htb->priv;
struct mlx5e_qos_node *root;
int err;
qos_dbg(htb->mdev, "TC_HTB_DESTROY\n");
/* Wait until real_num_tx_queues is updated for mlx5e_select_queue,
* so that we can safely switch to its non-HTB non-PTP fastpath.
*/
synchronize_net();
mlx5e_selq_prepare_htb(htb->selq, 0, 0);
mlx5e_selq_apply(htb->selq);
root = mlx5e_htb_node_find(htb, MLX5E_HTB_CLASSID_ROOT);
if (!root) {
qos_err(htb->mdev, "Failed to find the root node in the QoS tree\n");
return -ENOENT;
}
err = mlx5_qos_destroy_node(htb->mdev, root->hw_id);
if (err)
qos_err(htb->mdev, "Failed to destroy root node %u, err = %d\n",
root->hw_id, err);
mlx5e_htb_node_delete(htb, root);
mlx5e_qos_deactivate_all_queues(&priv->channels);
mlx5e_qos_close_all_queues(&priv->channels);
return err;
}
static int mlx5e_htb_convert_rate(struct mlx5e_htb *htb, u64 rate,
struct mlx5e_qos_node *parent, u32 *bw_share)
{
u64 share = 0;
while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw)
parent = parent->parent;
if (parent->max_average_bw)
share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT),
parent->max_average_bw);
else
share = 101;
*bw_share = share == 0 ? 1 : share > 100 ? 0 : share;
qos_dbg(htb->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n",
rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share);
return 0;
}
static void mlx5e_htb_convert_ceil(struct mlx5e_htb *htb, u64 ceil, u32 *max_average_bw)
{
/* Hardware treats 0 as "unlimited", set at least 1. */
*max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1);
qos_dbg(htb->mdev, "Convert: ceil %llu -> max_average_bw %u\n",
ceil, *max_average_bw);
}
static int
mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid,
u32 parent_classid, u64 rate, u64 ceil,
struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *node, *parent;
struct mlx5e_priv *priv = htb->priv;
int qid;
int err;
qos_dbg(htb->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n",
classid, parent_classid, rate, ceil);
qid = mlx5e_find_unused_qos_qid(htb);
if (qid < 0) {
NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached.");
return qid;
}
parent = mlx5e_htb_node_find(htb, parent_classid);
if (!parent)
return -EINVAL;
node = mlx5e_htb_node_create_leaf(htb, classid, qid, parent);
if (IS_ERR(node))
return PTR_ERR(node);
node->rate = rate;
mlx5e_htb_convert_rate(htb, rate, node->parent, &node->bw_share);
mlx5e_htb_convert_ceil(htb, ceil, &node->max_average_bw);
err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->hw_id,
node->bw_share, node->max_average_bw,
&node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node.");
qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n",
classid, err);
mlx5e_htb_node_delete(htb, node);
return err;
}
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, node);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n",
classid, err);
} else {
mlx5e_activate_qos_sq(priv, node);
}
}
return mlx5e_qid_from_qos(&priv->channels, node->qid);
}
static int
mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid,
u64 rate, u64 ceil, struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *node, *child;
struct mlx5e_priv *priv = htb->priv;
int err, tmp_err;
u32 new_hw_id;
u16 qid;
qos_dbg(htb->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n",
classid, child_classid, rate, ceil);
node = mlx5e_htb_node_find(htb, classid);
if (!node)
return -ENOENT;
err = mlx5_qos_create_inner_node(htb->mdev, node->parent->hw_id,
node->bw_share, node->max_average_bw,
&new_hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node.");
qos_err(htb->mdev, "Failed to create an inner node (class %04x), err = %d\n",
classid, err);
return err;
}
/* Intentionally reuse the qid for the upcoming first child. */
child = mlx5e_htb_node_create_leaf(htb, child_classid, node->qid, node);
if (IS_ERR(child)) {
err = PTR_ERR(child);
goto err_destroy_hw_node;
}
child->rate = rate;
mlx5e_htb_convert_rate(htb, rate, node, &child->bw_share);
mlx5e_htb_convert_ceil(htb, ceil, &child->max_average_bw);
err = mlx5_qos_create_leaf_node(htb->mdev, new_hw_id, child->bw_share,
child->max_average_bw, &child->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node.");
qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n",
classid, err);
goto err_delete_sw_node;
}
/* No fail point. */
qid = node->qid;
/* Pairs with mlx5e_htb_get_txq_by_classid. */
WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER);
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
mlx5e_deactivate_qos_sq(priv, qid);
mlx5e_close_qos_sq(priv, qid);
}
err = mlx5_qos_destroy_node(htb->mdev, node->hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, classid, err);
node->hw_id = new_hw_id;
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, child);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n",
classid, err);
} else {
mlx5e_activate_qos_sq(priv, child);
}
}
return 0;
err_delete_sw_node:
child->qid = MLX5E_QOS_QID_INNER;
mlx5e_htb_node_delete(htb, child);
err_destroy_hw_node:
tmp_err = mlx5_qos_destroy_node(htb->mdev, new_hw_id);
if (tmp_err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n",
new_hw_id, classid, tmp_err);
return err;
}
static struct mlx5e_qos_node *mlx5e_htb_node_find_by_qid(struct mlx5e_htb *htb, u16 qid)
{
struct mlx5e_qos_node *node = NULL;
int bkt;
hash_for_each(htb->qos_tc2node, bkt, node, hnode)
if (node->qid == qid)
break;
return node;
}
static void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq)
void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq)
{
qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid);
netdev_tx_reset_queue(txq);
netif_tx_start_queue(txq);
}
static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid)
void mlx5e_reset_qdisc(struct net_device *dev, u16 qid)
{
struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid);
struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
@ -782,290 +371,6 @@ static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid)
spin_unlock_bh(qdisc_lock(qdisc));
}
static int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid,
struct netlink_ext_ack *extack)
{
struct mlx5e_priv *priv = htb->priv;
struct mlx5e_qos_node *node;
struct netdev_queue *txq;
u16 qid, moved_qid;
bool opened;
int err;
qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid);
node = mlx5e_htb_node_find(htb, *classid);
if (!node)
return -ENOENT;
/* Store qid for reuse. */
qid = node->qid;
opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (opened) {
txq = netdev_get_tx_queue(htb->netdev,
mlx5e_qid_from_qos(&priv->channels, qid));
mlx5e_deactivate_qos_sq(priv, qid);
mlx5e_close_qos_sq(priv, qid);
}
err = mlx5_qos_destroy_node(htb->mdev, node->hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, *classid, err);
mlx5e_htb_node_delete(htb, node);
moved_qid = mlx5e_htb_cur_leaf_nodes(htb);
if (moved_qid == 0) {
/* The last QoS SQ was just destroyed. */
if (opened)
mlx5e_reactivate_qos_sq(priv, qid, txq);
return 0;
}
moved_qid--;
if (moved_qid < qid) {
/* The highest QoS SQ was just destroyed. */
WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u",
qid, moved_qid);
if (opened)
mlx5e_reactivate_qos_sq(priv, qid, txq);
return 0;
}
WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid);
qos_dbg(htb->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid);
node = mlx5e_htb_node_find_by_qid(htb, moved_qid);
WARN(!node, "Could not find a node with qid %u to move to queue %u",
moved_qid, qid);
/* Stop traffic to the old queue. */
WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER);
__clear_bit(moved_qid, priv->htb->qos_used_qids);
if (opened) {
txq = netdev_get_tx_queue(htb->netdev,
mlx5e_qid_from_qos(&priv->channels, moved_qid));
mlx5e_deactivate_qos_sq(priv, moved_qid);
mlx5e_close_qos_sq(priv, moved_qid);
}
/* Prevent packets from the old class from getting into the new one. */
mlx5e_reset_qdisc(htb->netdev, moved_qid);
__set_bit(qid, htb->qos_used_qids);
WRITE_ONCE(node->qid, qid);
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, node);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n",
node->classid, moved_qid, qid, err);
} else {
mlx5e_activate_qos_sq(priv, node);
}
}
mlx5e_update_tx_netdev_queues(priv);
if (opened)
mlx5e_reactivate_qos_sq(priv, moved_qid, txq);
*classid = node->classid;
return 0;
}
static int
mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force,
struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *node, *parent;
struct mlx5e_priv *priv = htb->priv;
u32 old_hw_id, new_hw_id;
int err, saved_err = 0;
u16 qid;
qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n",
force ? "_FORCE" : "", classid);
node = mlx5e_htb_node_find(htb, classid);
if (!node)
return -ENOENT;
err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->parent->hw_id,
node->parent->bw_share,
node->parent->max_average_bw,
&new_hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node.");
qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n",
classid, err);
if (!force)
return err;
saved_err = err;
}
/* Store qid for reuse and prevent clearing the bit. */
qid = node->qid;
/* Pairs with mlx5e_htb_get_txq_by_classid. */
WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER);
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
mlx5e_deactivate_qos_sq(priv, qid);
mlx5e_close_qos_sq(priv, qid);
}
/* Prevent packets from the old class from getting into the new one. */
mlx5e_reset_qdisc(htb->netdev, qid);
err = mlx5_qos_destroy_node(htb->mdev, node->hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, classid, err);
parent = node->parent;
mlx5e_htb_node_delete(htb, node);
node = parent;
WRITE_ONCE(node->qid, qid);
/* Early return on error in force mode. Parent will still be an inner
* node to be deleted by a following delete operation.
*/
if (saved_err)
return saved_err;
old_hw_id = node->hw_id;
node->hw_id = new_hw_id;
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
err = mlx5e_open_qos_sq(priv, &priv->channels, node);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ.");
qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n",
classid, err);
} else {
mlx5e_activate_qos_sq(priv, node);
}
}
err = mlx5_qos_destroy_node(htb->mdev, old_hw_id);
if (err) /* Not fatal. */
qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n",
node->hw_id, classid, err);
return 0;
}
static int
mlx5e_htb_update_children(struct mlx5e_htb *htb, struct mlx5e_qos_node *node,
struct netlink_ext_ack *extack)
{
struct mlx5e_qos_node *child;
int err = 0;
int bkt;
hash_for_each(htb->qos_tc2node, bkt, child, hnode) {
u32 old_bw_share = child->bw_share;
int err_one;
if (child->parent != node)
continue;
mlx5e_htb_convert_rate(htb, child->rate, node, &child->bw_share);
if (child->bw_share == old_bw_share)
continue;
err_one = mlx5_qos_update_node(htb->mdev, child->hw_id, child->bw_share,
child->max_average_bw, child->hw_id);
if (!err && err_one) {
err = err_one;
NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node.");
qos_err(htb->mdev, "Failed to modify a child node (class %04x), err = %d\n",
node->classid, err);
}
}
return err;
}
static int
mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil,
struct netlink_ext_ack *extack)
{
u32 bw_share, max_average_bw;
struct mlx5e_qos_node *node;
bool ceil_changed = false;
int err;
qos_dbg(htb->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n",
classid, rate, ceil);
node = mlx5e_htb_node_find(htb, classid);
if (!node)
return -ENOENT;
node->rate = rate;
mlx5e_htb_convert_rate(htb, rate, node->parent, &bw_share);
mlx5e_htb_convert_ceil(htb, ceil, &max_average_bw);
err = mlx5_qos_update_node(htb->mdev, node->parent->hw_id, bw_share,
max_average_bw, node->hw_id);
if (err) {
NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node.");
qos_err(htb->mdev, "Failed to modify a node (class %04x), err = %d\n",
classid, err);
return err;
}
if (max_average_bw != node->max_average_bw)
ceil_changed = true;
node->bw_share = bw_share;
node->max_average_bw = max_average_bw;
if (ceil_changed)
err = mlx5e_htb_update_children(htb, node, extack);
return err;
}
/* HTB API */
static struct mlx5e_htb *mlx5e_htb_alloc(void)
{
return kvzalloc(sizeof(struct mlx5e_htb), GFP_KERNEL);
}
static void mlx5e_htb_free(struct mlx5e_htb *htb)
{
kvfree(htb);
}
/* HTB API */
static int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt,
struct net_device *netdev, struct mlx5_core_dev *mdev,
struct mlx5e_selq *selq, struct mlx5e_priv *priv)
{
htb->mdev = mdev;
htb->netdev = netdev;
htb->selq = selq;
htb->priv = priv;
hash_init(htb->qos_tc2node);
return mlx5e_htb_root_add(htb, htb_qopt->parent_classid, htb_qopt->classid,
htb_qopt->extack);
}
static void mlx5e_htb_cleanup(struct mlx5e_htb *htb)
{
mlx5e_htb_root_del(htb);
}
int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt)
{
struct mlx5e_htb *htb = priv->htb;

20
drivers/net/ethernet/mellanox/mlx5/core/en/qos.h
View File

@ -6,7 +6,7 @@
#include <linux/mlx5/driver.h>
#define MLX5E_QOS_MAX_LEAF_NODES 256
#define BYTES_IN_MBIT 125000
struct mlx5e_priv;
struct mlx5e_htb;
@ -17,17 +17,27 @@ struct tc_htb_qopt_offload;
int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes);
int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev);
/* TX datapath API */
int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid);
/* SQ lifecycle */
int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs,
u16 node_qid, u32 hw_id);
int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id);
void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid);
void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid);
void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq);
void mlx5e_reset_qdisc(struct net_device *dev, u16 qid);
int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs);
void mlx5e_qos_activate_queues(struct mlx5e_priv *priv);
void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c);
void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs);
void mlx5e_qos_close_queues(struct mlx5e_channel *c);
void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs);
int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs);
/* TX datapath API */
u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid);
/* HTB API */
int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb);
int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb);
/* MQPRIO TX rate limit */

1
drivers/net/ethernet/mellanox/mlx5/core/en/selq.c
View File

@ -7,6 +7,7 @@
#include <linux/rcupdate.h>
#include "en.h"
#include "en/ptp.h"
#include "en/htb.h"
struct mlx5e_selq_params {
unsigned int num_regular_queues;

1
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View File

@ -63,6 +63,7 @@
#include "en/devlink.h"
#include "lib/mlx5.h"
#include "en/ptp.h"
#include "en/htb.h"
#include "qos.h"
#include "en/trap.h"