linux/net/sched/sch_cbs.c
Vinicius Costa Gomes 3e8b9bfa11 net/sched: cbs: Fix not adding cbs instance to list
When removing a cbs instance when offloading is enabled, the crash
below can be observed.

The problem happens because that when offloading is enabled, the cbs
instance is not added to the list.

Also, the current code doesn't handle correctly the case when offload
is disabled without removing the qdisc: if the link speed changes the
credit calculations will be wrong. When we create the cbs instance
with offloading enabled, it's not added to the notification list, when
later we disable offloading, it's not in the list, so link speed
changes will not affect it.

The solution for both issues is the same, add the cbs instance being
created unconditionally to the global list, even if the link state
notification isn't useful "right now".

Crash log:

[518758.189866] BUG: kernel NULL pointer dereference, address: 0000000000000000
[518758.189870] #PF: supervisor read access in kernel mode
[518758.189871] #PF: error_code(0x0000) - not-present page
[518758.189872] PGD 0 P4D 0
[518758.189874] Oops: 0000 [#1] SMP PTI
[518758.189876] CPU: 3 PID: 4825 Comm: tc Not tainted 5.2.9 #1
[518758.189877] Hardware name: Gigabyte Technology Co., Ltd. Z390 AORUS ULTRA/Z390 AORUS ULTRA-CF, BIOS F7 03/14/2019
[518758.189881] RIP: 0010:__list_del_entry_valid+0x29/0xa0
[518758.189883] Code: 90 48 b8 00 01 00 00 00 00 ad de 55 48 8b 17 4c 8b 47 08 48 89 e5 48 39 c2 74 27 48 b8 00 02 00 00 00 00 ad de 49 39 c0 74 2d <49> 8b 30 48 39 fe 75 3d 48 8b 52 08 48 39 f2 75 4c b8 01 00 00 00
[518758.189885] RSP: 0018:ffffa27e43903990 EFLAGS: 00010207
[518758.189887] RAX: dead000000000200 RBX: ffff8bce69f0f000 RCX: 0000000000000000
[518758.189888] RDX: 0000000000000000 RSI: ffff8bce69f0f064 RDI: ffff8bce69f0f1e0
[518758.189890] RBP: ffffa27e43903990 R08: 0000000000000000 R09: ffff8bce69e788c0
[518758.189891] R10: ffff8bce62acd400 R11: 00000000000003cb R12: ffff8bce69e78000
[518758.189892] R13: ffff8bce69f0f140 R14: 0000000000000000 R15: 0000000000000000
[518758.189894] FS:  00007fa1572c8f80(0000) GS:ffff8bce6e0c0000(0000) knlGS:0000000000000000
[518758.189895] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[518758.189896] CR2: 0000000000000000 CR3: 000000040a398006 CR4: 00000000003606e0
[518758.189898] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[518758.189899] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[518758.189900] Call Trace:
[518758.189904]  cbs_destroy+0x32/0xa0 [sch_cbs]
[518758.189906]  qdisc_destroy+0x45/0x120
[518758.189907]  qdisc_put+0x25/0x30
[518758.189908]  qdisc_graft+0x2c1/0x450
[518758.189910]  tc_get_qdisc+0x1c8/0x310
[518758.189912]  ? get_page_from_freelist+0x91a/0xcb0
[518758.189914]  rtnetlink_rcv_msg+0x293/0x360
[518758.189916]  ? kmem_cache_alloc_node_trace+0x178/0x260
[518758.189918]  ? __kmalloc_node_track_caller+0x38/0x50
[518758.189920]  ? rtnl_calcit.isra.0+0xf0/0xf0
[518758.189922]  netlink_rcv_skb+0x48/0x110
[518758.189923]  rtnetlink_rcv+0x10/0x20
[518758.189925]  netlink_unicast+0x15b/0x1d0
[518758.189926]  netlink_sendmsg+0x1ea/0x380
[518758.189929]  sock_sendmsg+0x2f/0x40
[518758.189930]  ___sys_sendmsg+0x295/0x2f0
[518758.189932]  ? ___sys_recvmsg+0x151/0x1e0
[518758.189933]  ? do_wp_page+0x7e/0x450
[518758.189935]  __sys_sendmsg+0x48/0x80
[518758.189937]  __x64_sys_sendmsg+0x1a/0x20
[518758.189939]  do_syscall_64+0x53/0x1f0
[518758.189941]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
[518758.189942] RIP: 0033:0x7fa15755169a
[518758.189944] Code: 48 c7 c0 ff ff ff ff eb be 0f 1f 80 00 00 00 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 18 b8 2e 00 00 00 c5 fc 77 0f 05 <48> 3d 00 f0 ff ff 77 5e c3 0f 1f 44 00 00 48 83 ec 28 89 54 24 1c
[518758.189946] RSP: 002b:00007ffda58b60b8 EFLAGS: 00000246 ORIG_RAX: 000000000000002e
[518758.189948] RAX: ffffffffffffffda RBX: 000055e4b836d9a0 RCX: 00007fa15755169a
[518758.189949] RDX: 0000000000000000 RSI: 00007ffda58b6128 RDI: 0000000000000003
[518758.189951] RBP: 00007ffda58b6190 R08: 0000000000000001 R09: 000055e4b9d848a0
[518758.189952] R10: 0000000000000000 R11: 0000000000000246 R12: 000000005d654b49
[518758.189953] R13: 0000000000000000 R14: 00007ffda58b6230 R15: 00007ffda58b6210
[518758.189955] Modules linked in: sch_cbs sch_etf sch_mqprio netlink_diag unix_diag e1000e igb intel_pch_thermal thermal video backlight pcc_cpufreq
[518758.189960] CR2: 0000000000000000
[518758.189961] ---[ end trace 6a13f7aaf5376019 ]---
[518758.189963] RIP: 0010:__list_del_entry_valid+0x29/0xa0
[518758.189964] Code: 90 48 b8 00 01 00 00 00 00 ad de 55 48 8b 17 4c 8b 47 08 48 89 e5 48 39 c2 74 27 48 b8 00 02 00 00 00 00 ad de 49 39 c0 74 2d <49> 8b 30 48 39 fe 75 3d 48 8b 52 08 48 39 f2 75 4c b8 01 00 00 00
[518758.189967] RSP: 0018:ffffa27e43903990 EFLAGS: 00010207
[518758.189968] RAX: dead000000000200 RBX: ffff8bce69f0f000 RCX: 0000000000000000
[518758.189969] RDX: 0000000000000000 RSI: ffff8bce69f0f064 RDI: ffff8bce69f0f1e0
[518758.189971] RBP: ffffa27e43903990 R08: 0000000000000000 R09: ffff8bce69e788c0
[518758.189972] R10: ffff8bce62acd400 R11: 00000000000003cb R12: ffff8bce69e78000
[518758.189973] R13: ffff8bce69f0f140 R14: 0000000000000000 R15: 0000000000000000
[518758.189975] FS:  00007fa1572c8f80(0000) GS:ffff8bce6e0c0000(0000) knlGS:0000000000000000
[518758.189976] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[518758.189977] CR2: 0000000000000000 CR3: 000000040a398006 CR4: 00000000003606e0
[518758.189979] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[518758.189980] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

Fixes: e0a7683d30 ("net/sched: cbs: fix port_rate miscalculation")
Signed-off-by: Vinicius Costa Gomes <vinicius.gomes@intel.com>
Acked-by: Cong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-09-26 09:03:03 +02:00

572 lines
13 KiB
C

// SPDX-License-Identifier: GPL-2.0-or-later
/*
* net/sched/sch_cbs.c Credit Based Shaper
*
* Authors: Vinicius Costa Gomes <vinicius.gomes@intel.com>
*/
/* Credit Based Shaper (CBS)
* =========================
*
* This is a simple rate-limiting shaper aimed at TSN applications on
* systems with known traffic workloads.
*
* Its algorithm is defined by the IEEE 802.1Q-2014 Specification,
* Section 8.6.8.2, and explained in more detail in the Annex L of the
* same specification.
*
* There are four tunables to be considered:
*
* 'idleslope': Idleslope is the rate of credits that is
* accumulated (in kilobits per second) when there is at least
* one packet waiting for transmission. Packets are transmitted
* when the current value of credits is equal or greater than
* zero. When there is no packet to be transmitted the amount of
* credits is set to zero. This is the main tunable of the CBS
* algorithm.
*
* 'sendslope':
* Sendslope is the rate of credits that is depleted (it should be a
* negative number of kilobits per second) when a transmission is
* ocurring. It can be calculated as follows, (IEEE 802.1Q-2014 Section
* 8.6.8.2 item g):
*
* sendslope = idleslope - port_transmit_rate
*
* 'hicredit': Hicredit defines the maximum amount of credits (in
* bytes) that can be accumulated. Hicredit depends on the
* characteristics of interfering traffic,
* 'max_interference_size' is the maximum size of any burst of
* traffic that can delay the transmission of a frame that is
* available for transmission for this traffic class, (IEEE
* 802.1Q-2014 Annex L, Equation L-3):
*
* hicredit = max_interference_size * (idleslope / port_transmit_rate)
*
* 'locredit': Locredit is the minimum amount of credits that can
* be reached. It is a function of the traffic flowing through
* this qdisc (IEEE 802.1Q-2014 Annex L, Equation L-2):
*
* locredit = max_frame_size * (sendslope / port_transmit_rate)
*/
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <net/netevent.h>
#include <net/netlink.h>
#include <net/sch_generic.h>
#include <net/pkt_sched.h>
static LIST_HEAD(cbs_list);
static DEFINE_SPINLOCK(cbs_list_lock);
#define BYTES_PER_KBIT (1000LL / 8)
struct cbs_sched_data {
bool offload;
int queue;
atomic64_t port_rate; /* in bytes/s */
s64 last; /* timestamp in ns */
s64 credits; /* in bytes */
s32 locredit; /* in bytes */
s32 hicredit; /* in bytes */
s64 sendslope; /* in bytes/s */
s64 idleslope; /* in bytes/s */
struct qdisc_watchdog watchdog;
int (*enqueue)(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free);
struct sk_buff *(*dequeue)(struct Qdisc *sch);
struct Qdisc *qdisc;
struct list_head cbs_list;
};
static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct Qdisc *child,
struct sk_buff **to_free)
{
unsigned int len = qdisc_pkt_len(skb);
int err;
err = child->ops->enqueue(skb, child, to_free);
if (err != NET_XMIT_SUCCESS)
return err;
sch->qstats.backlog += len;
sch->q.qlen++;
return NET_XMIT_SUCCESS;
}
static int cbs_enqueue_offload(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct Qdisc *qdisc = q->qdisc;
return cbs_child_enqueue(skb, sch, qdisc, to_free);
}
static int cbs_enqueue_soft(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct Qdisc *qdisc = q->qdisc;
if (sch->q.qlen == 0 && q->credits > 0) {
/* We need to stop accumulating credits when there's
* no enqueued packets and q->credits is positive.
*/
q->credits = 0;
q->last = ktime_get_ns();
}
return cbs_child_enqueue(skb, sch, qdisc, to_free);
}
static int cbs_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
struct cbs_sched_data *q = qdisc_priv(sch);
return q->enqueue(skb, sch, to_free);
}
/* timediff is in ns, slope is in bytes/s */
static s64 timediff_to_credits(s64 timediff, s64 slope)
{
return div64_s64(timediff * slope, NSEC_PER_SEC);
}
static s64 delay_from_credits(s64 credits, s64 slope)
{
if (unlikely(slope == 0))
return S64_MAX;
return div64_s64(-credits * NSEC_PER_SEC, slope);
}
static s64 credits_from_len(unsigned int len, s64 slope, s64 port_rate)
{
if (unlikely(port_rate == 0))
return S64_MAX;
return div64_s64(len * slope, port_rate);
}
static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child)
{
struct sk_buff *skb;
skb = child->ops->dequeue(child);
if (!skb)
return NULL;
qdisc_qstats_backlog_dec(sch, skb);
qdisc_bstats_update(sch, skb);
sch->q.qlen--;
return skb;
}
static struct sk_buff *cbs_dequeue_soft(struct Qdisc *sch)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct Qdisc *qdisc = q->qdisc;
s64 now = ktime_get_ns();
struct sk_buff *skb;
s64 credits;
int len;
if (q->credits < 0) {
credits = timediff_to_credits(now - q->last, q->idleslope);
credits = q->credits + credits;
q->credits = min_t(s64, credits, q->hicredit);
if (q->credits < 0) {
s64 delay;
delay = delay_from_credits(q->credits, q->idleslope);
qdisc_watchdog_schedule_ns(&q->watchdog, now + delay);
q->last = now;
return NULL;
}
}
skb = cbs_child_dequeue(sch, qdisc);
if (!skb)
return NULL;
len = qdisc_pkt_len(skb);
/* As sendslope is a negative number, this will decrease the
* amount of q->credits.
*/
credits = credits_from_len(len, q->sendslope,
atomic64_read(&q->port_rate));
credits += q->credits;
q->credits = max_t(s64, credits, q->locredit);
q->last = now;
return skb;
}
static struct sk_buff *cbs_dequeue_offload(struct Qdisc *sch)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct Qdisc *qdisc = q->qdisc;
return cbs_child_dequeue(sch, qdisc);
}
static struct sk_buff *cbs_dequeue(struct Qdisc *sch)
{
struct cbs_sched_data *q = qdisc_priv(sch);
return q->dequeue(sch);
}
static const struct nla_policy cbs_policy[TCA_CBS_MAX + 1] = {
[TCA_CBS_PARMS] = { .len = sizeof(struct tc_cbs_qopt) },
};
static void cbs_disable_offload(struct net_device *dev,
struct cbs_sched_data *q)
{
struct tc_cbs_qopt_offload cbs = { };
const struct net_device_ops *ops;
int err;
if (!q->offload)
return;
q->enqueue = cbs_enqueue_soft;
q->dequeue = cbs_dequeue_soft;
ops = dev->netdev_ops;
if (!ops->ndo_setup_tc)
return;
cbs.queue = q->queue;
cbs.enable = 0;
err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_CBS, &cbs);
if (err < 0)
pr_warn("Couldn't disable CBS offload for queue %d\n",
cbs.queue);
}
static int cbs_enable_offload(struct net_device *dev, struct cbs_sched_data *q,
const struct tc_cbs_qopt *opt,
struct netlink_ext_ack *extack)
{
const struct net_device_ops *ops = dev->netdev_ops;
struct tc_cbs_qopt_offload cbs = { };
int err;
if (!ops->ndo_setup_tc) {
NL_SET_ERR_MSG(extack, "Specified device does not support cbs offload");
return -EOPNOTSUPP;
}
cbs.queue = q->queue;
cbs.enable = 1;
cbs.hicredit = opt->hicredit;
cbs.locredit = opt->locredit;
cbs.idleslope = opt->idleslope;
cbs.sendslope = opt->sendslope;
err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_CBS, &cbs);
if (err < 0) {
NL_SET_ERR_MSG(extack, "Specified device failed to setup cbs hardware offload");
return err;
}
q->enqueue = cbs_enqueue_offload;
q->dequeue = cbs_dequeue_offload;
return 0;
}
static void cbs_set_port_rate(struct net_device *dev, struct cbs_sched_data *q)
{
struct ethtool_link_ksettings ecmd;
int speed = SPEED_10;
int port_rate;
int err;
err = __ethtool_get_link_ksettings(dev, &ecmd);
if (err < 0)
goto skip;
if (ecmd.base.speed != SPEED_UNKNOWN)
speed = ecmd.base.speed;
skip:
port_rate = speed * 1000 * BYTES_PER_KBIT;
atomic64_set(&q->port_rate, port_rate);
netdev_dbg(dev, "cbs: set %s's port_rate to: %lld, linkspeed: %d\n",
dev->name, (long long)atomic64_read(&q->port_rate),
ecmd.base.speed);
}
static int cbs_dev_notifier(struct notifier_block *nb, unsigned long event,
void *ptr)
{
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
struct cbs_sched_data *q;
struct net_device *qdev;
bool found = false;
ASSERT_RTNL();
if (event != NETDEV_UP && event != NETDEV_CHANGE)
return NOTIFY_DONE;
spin_lock(&cbs_list_lock);
list_for_each_entry(q, &cbs_list, cbs_list) {
qdev = qdisc_dev(q->qdisc);
if (qdev == dev) {
found = true;
break;
}
}
spin_unlock(&cbs_list_lock);
if (found)
cbs_set_port_rate(dev, q);
return NOTIFY_DONE;
}
static int cbs_change(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct net_device *dev = qdisc_dev(sch);
struct nlattr *tb[TCA_CBS_MAX + 1];
struct tc_cbs_qopt *qopt;
int err;
err = nla_parse_nested_deprecated(tb, TCA_CBS_MAX, opt, cbs_policy,
extack);
if (err < 0)
return err;
if (!tb[TCA_CBS_PARMS]) {
NL_SET_ERR_MSG(extack, "Missing CBS parameter which are mandatory");
return -EINVAL;
}
qopt = nla_data(tb[TCA_CBS_PARMS]);
if (!qopt->offload) {
cbs_set_port_rate(dev, q);
cbs_disable_offload(dev, q);
} else {
err = cbs_enable_offload(dev, q, qopt, extack);
if (err < 0)
return err;
}
/* Everything went OK, save the parameters used. */
q->hicredit = qopt->hicredit;
q->locredit = qopt->locredit;
q->idleslope = qopt->idleslope * BYTES_PER_KBIT;
q->sendslope = qopt->sendslope * BYTES_PER_KBIT;
q->offload = qopt->offload;
return 0;
}
static int cbs_init(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct net_device *dev = qdisc_dev(sch);
if (!opt) {
NL_SET_ERR_MSG(extack, "Missing CBS qdisc options which are mandatory");
return -EINVAL;
}
q->qdisc = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
sch->handle, extack);
if (!q->qdisc)
return -ENOMEM;
spin_lock(&cbs_list_lock);
list_add(&q->cbs_list, &cbs_list);
spin_unlock(&cbs_list_lock);
qdisc_hash_add(q->qdisc, false);
q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0);
q->enqueue = cbs_enqueue_soft;
q->dequeue = cbs_dequeue_soft;
qdisc_watchdog_init(&q->watchdog, sch);
return cbs_change(sch, opt, extack);
}
static void cbs_destroy(struct Qdisc *sch)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct net_device *dev = qdisc_dev(sch);
/* Nothing to do if we couldn't create the underlying qdisc */
if (!q->qdisc)
return;
qdisc_watchdog_cancel(&q->watchdog);
cbs_disable_offload(dev, q);
spin_lock(&cbs_list_lock);
list_del(&q->cbs_list);
spin_unlock(&cbs_list_lock);
qdisc_put(q->qdisc);
}
static int cbs_dump(struct Qdisc *sch, struct sk_buff *skb)
{
struct cbs_sched_data *q = qdisc_priv(sch);
struct tc_cbs_qopt opt = { };
struct nlattr *nest;
nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
if (!nest)
goto nla_put_failure;
opt.hicredit = q->hicredit;
opt.locredit = q->locredit;
opt.sendslope = div64_s64(q->sendslope, BYTES_PER_KBIT);
opt.idleslope = div64_s64(q->idleslope, BYTES_PER_KBIT);
opt.offload = q->offload;
if (nla_put(skb, TCA_CBS_PARMS, sizeof(opt), &opt))
goto nla_put_failure;
return nla_nest_end(skb, nest);
nla_put_failure:
nla_nest_cancel(skb, nest);
return -1;
}
static int cbs_dump_class(struct Qdisc *sch, unsigned long cl,
struct sk_buff *skb, struct tcmsg *tcm)
{
struct cbs_sched_data *q = qdisc_priv(sch);
if (cl != 1 || !q->qdisc) /* only one class */
return -ENOENT;
tcm->tcm_handle |= TC_H_MIN(1);
tcm->tcm_info = q->qdisc->handle;
return 0;
}
static int cbs_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
struct Qdisc **old, struct netlink_ext_ack *extack)
{
struct cbs_sched_data *q = qdisc_priv(sch);
if (!new) {
new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
sch->handle, NULL);
if (!new)
new = &noop_qdisc;
}
*old = qdisc_replace(sch, new, &q->qdisc);
return 0;
}
static struct Qdisc *cbs_leaf(struct Qdisc *sch, unsigned long arg)
{
struct cbs_sched_data *q = qdisc_priv(sch);
return q->qdisc;
}
static unsigned long cbs_find(struct Qdisc *sch, u32 classid)
{
return 1;
}
static void cbs_walk(struct Qdisc *sch, struct qdisc_walker *walker)
{
if (!walker->stop) {
if (walker->count >= walker->skip) {
if (walker->fn(sch, 1, walker) < 0) {
walker->stop = 1;
return;
}
}
walker->count++;
}
}
static const struct Qdisc_class_ops cbs_class_ops = {
.graft = cbs_graft,
.leaf = cbs_leaf,
.find = cbs_find,
.walk = cbs_walk,
.dump = cbs_dump_class,
};
static struct Qdisc_ops cbs_qdisc_ops __read_mostly = {
.id = "cbs",
.cl_ops = &cbs_class_ops,
.priv_size = sizeof(struct cbs_sched_data),
.enqueue = cbs_enqueue,
.dequeue = cbs_dequeue,
.peek = qdisc_peek_dequeued,
.init = cbs_init,
.reset = qdisc_reset_queue,
.destroy = cbs_destroy,
.change = cbs_change,
.dump = cbs_dump,
.owner = THIS_MODULE,
};
static struct notifier_block cbs_device_notifier = {
.notifier_call = cbs_dev_notifier,
};
static int __init cbs_module_init(void)
{
int err;
err = register_netdevice_notifier(&cbs_device_notifier);
if (err)
return err;
err = register_qdisc(&cbs_qdisc_ops);
if (err)
unregister_netdevice_notifier(&cbs_device_notifier);
return err;
}
static void __exit cbs_module_exit(void)
{
unregister_qdisc(&cbs_qdisc_ops);
unregister_netdevice_notifier(&cbs_device_notifier);
}
module_init(cbs_module_init)
module_exit(cbs_module_exit)
MODULE_LICENSE("GPL");