Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-24 21:21:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

net: rps: add rps_input_queue_head_add() helper

Browse Source 
process_backlog() can batch increments of sd->input_queue_head,
saving some memory bandwidth.

Also add READ_ONCE()/WRITE_ONCE() annotations around
sd->input_queue_head accesses.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2024-03-29 15:42:24 +00:00 committed by David S. Miller
parent 36b83ffcf2
commit c62fdf5b11
2 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/net/rps.h
View File

@ -138,11 +138,16 @@ static inline void rps_input_queue_tail_save(u32 *dest, u32 tail)
#endif #endif
} }
static inline void rps_input_queue_head_incr(struct softnet_data *sd) static inline void rps_input_queue_head_add(struct softnet_data *sd, int val)
{ {
#ifdef CONFIG_RPS #ifdef CONFIG_RPS
sd->input_queue_head++; WRITE_ONCE(sd->input_queue_head, sd->input_queue_head + val);
#endif #endif
} }
static inline void rps_input_queue_head_incr(struct softnet_data *sd)
{
rps_input_queue_head_add(sd, 1);
}
#endif /* _NET_RPS_H */ #endif /* _NET_RPS_H */

13
net/core/dev.c
View File

@ -4528,7 +4528,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
out: out:
#endif #endif
rflow->last_qtail = rflow->last_qtail =
per_cpu(softnet_data, next_cpu).input_queue_head; READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);
} }
rflow->cpu = next_cpu; rflow->cpu = next_cpu;
@ -4610,7 +4610,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
*/ */
if (unlikely(tcpu != next_cpu) && if (unlikely(tcpu != next_cpu) &&
(tcpu >= nr_cpu_ids || !cpu_online(tcpu) || (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
((int)(per_cpu(softnet_data, tcpu).input_queue_head - ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -
READ_ONCE(rflow->last_qtail))) >= 0)) { READ_ONCE(rflow->last_qtail))) >= 0)) {
tcpu = next_cpu; tcpu = next_cpu;
rflow = set_rps_cpu(dev, skb, rflow, next_cpu); rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
@ -4665,7 +4665,7 @@ bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
rflow = &flow_table->flows[flow_id]; rflow = &flow_table->flows[flow_id];
cpu = READ_ONCE(rflow->cpu); cpu = READ_ONCE(rflow->cpu);
if (rflow->filter == filter_id && cpu < nr_cpu_ids && if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
((int)(per_cpu(softnet_data, cpu).input_queue_head - ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) -
READ_ONCE(rflow->last_qtail)) < READ_ONCE(rflow->last_qtail)) <
(int)(10 * flow_table->mask))) (int)(10 * flow_table->mask)))
expire = false; expire = false;
@ -6045,9 +6045,10 @@ static int process_backlog(struct napi_struct *napi, int quota)
rcu_read_lock(); rcu_read_lock();
__netif_receive_skb(skb); __netif_receive_skb(skb);
rcu_read_unlock(); rcu_read_unlock();
rps_input_queue_head_incr(sd); if (++work >= quota) {
if (++work >= quota) rps_input_queue_head_add(sd, work);
return work; return work;
}
} }
@ -6070,6 +6071,8 @@ static int process_backlog(struct napi_struct *napi, int quota)
backlog_unlock_irq_enable(sd); backlog_unlock_irq_enable(sd);
} }
if (work)
rps_input_queue_head_add(sd, work);
return work; return work;
} }