leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

net: add per-cpu storage and net->core_stats

Browse Source 
Before adding yet another possibly contended atomic_long_t,
it is time to add per-cpu storage for existing ones:
 dev->tx_dropped, dev->rx_dropped, and dev->rx_nohandler

Because many devices do not have to increment such counters,
allocate the per-cpu storage on demand, so that dev_get_stats()
does not have to spend considerable time folding zero counters.

Note that some drivers have abused these counters which
were supposed to be only used by core networking stack.

v4: should use per_cpu_ptr() in dev_get_stats() (Jakub)
v3: added a READ_ONCE() in netdev_core_stats_alloc() (Paolo)
v2: add a missing include (reported by kernel test robot <lkp@intel.com>)
    Change in netdev_core_stats_alloc() (Jakub)

Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: jeffreyji <jeffreyji@google.com>
Reviewed-by: Brian Vazquez <brianvv@google.com>
Reviewed-by: Jakub Kicinski <kuba@kernel.org>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Link: https://lore.kernel.org/r/20220311051420.2608812-1-eric.dumazet@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet
2022-03-10 21:14:20 -08:00
committed by Jakub Kicinski
parent a8c063372b
commit 625788b584
15 changed files with 101 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
net/core/dev.c
View File

@@ -3633,7 +3633,7 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device
out_kfree_skb:
kfree_skb(skb);
out_null:
atomic_long_inc(&dev->tx_dropped);
dev_core_stats_tx_dropped_inc(dev);
return NULL;
}
@@ -4184,7 +4184,7 @@ recursion_alert:
rc = -ENETDOWN;
rcu_read_unlock_bh();
atomic_long_inc(&dev->tx_dropped);
dev_core_stats_tx_dropped_inc(dev);
kfree_skb_list(skb);
return rc;
out:
@@ -4236,7 +4236,7 @@ int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
local_bh_enable();
return ret;
drop:
atomic_long_inc(&dev->tx_dropped);
dev_core_stats_tx_dropped_inc(dev);
kfree_skb_list(skb);
return NET_XMIT_DROP;
}
@@ -4602,7 +4602,7 @@ drop:
sd->dropped++;
rps_unlock_irq_restore(sd, &flags);
atomic_long_inc(&skb->dev->rx_dropped);
dev_core_stats_rx_dropped_inc(skb->dev);
kfree_skb_reason(skb, reason);
return NET_RX_DROP;
}
@@ -5357,10 +5357,10 @@ check_vlan_id:
} else {
drop:
if (!deliver_exact) {
atomic_long_inc(&skb->dev->rx_dropped);
dev_core_stats_rx_dropped_inc(skb->dev);
kfree_skb_reason(skb, SKB_DROP_REASON_PTYPE_ABSENT);
} else {
atomic_long_inc(&skb->dev->rx_nohandler);
dev_core_stats_rx_nohandler_inc(skb->dev);
kfree_skb(skb);
}
/* Jamal, now you will not able to escape explaining
@@ -10280,6 +10280,25 @@ void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
}
EXPORT_SYMBOL(netdev_stats_to_stats64);
struct net_device_core_stats *netdev_core_stats_alloc(struct net_device *dev)
{
struct net_device_core_stats __percpu *p;
p = alloc_percpu_gfp(struct net_device_core_stats,
GFP_ATOMIC | __GFP_NOWARN);
if (p && cmpxchg(&dev->core_stats, NULL, p))
free_percpu(p);
/* This READ_ONCE() pairs with the cmpxchg() above */
p = READ_ONCE(dev->core_stats);
if (!p)
return NULL;
return this_cpu_ptr(p);
}
EXPORT_SYMBOL(netdev_core_stats_alloc);
/**
* dev_get_stats - get network device statistics
* @dev: device to get statistics from
@@ -10294,6 +10313,7 @@ struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
struct rtnl_link_stats64 *storage)
{
const struct net_device_ops *ops = dev->netdev_ops;
const struct net_device_core_stats __percpu *p;
if (ops->ndo_get_stats64) {
memset(storage, 0, sizeof(*storage));
@@ -10303,9 +10323,20 @@ struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
} else {
netdev_stats_to_stats64(storage, &dev->stats);
}
storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
p = READ_ONCE(dev->core_stats);
if (p) {
const struct net_device_core_stats *core_stats;
int i;
for_each_possible_cpu(i) {
core_stats = per_cpu_ptr(p, i);
storage->rx_dropped += local_read(&core_stats->rx_dropped);
storage->tx_dropped += local_read(&core_stats->tx_dropped);
storage->rx_nohandler += local_read(&core_stats->rx_nohandler);
}
}
return storage;
}
EXPORT_SYMBOL(dev_get_stats);
@@ -10567,6 +10598,8 @@ void free_netdev(struct net_device *dev)
free_percpu(dev->pcpu_refcnt);
dev->pcpu_refcnt = NULL;
#endif
free_percpu(dev->core_stats);
dev->core_stats = NULL;
free_percpu(dev->xdp_bulkq);
dev->xdp_bulkq = NULL;