leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]()

Browse Source 
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.

Typical usage scenarios:

        #include <linux/static_key.h>

        struct static_key key = STATIC_KEY_INIT_TRUE;

        if (static_key_false(&key))
                do unlikely code
        else
                do likely code

Or:

        if (static_key_true(&key))
                do likely code
        else
                do unlikely code

The static key is modified via:

        static_key_slow_inc(&key);
        ...
        static_key_slow_dec(&key);

The 'slow' prefix makes it abundantly clear that this is an
expensive operation.

I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.

On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Jason Baron <jbaron@redhat.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: a.p.zijlstra@chello.nl
Cc: mathieu.desnoyers@efficios.com
Cc: davem@davemloft.net
Cc: ddaney.cavm@gmail.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20120222085809.GA26397@elte.hu
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Ingo Molnar
2012-02-24 08:31:31 +01:00
parent 1cfa60dc7d
commit c5905afb0e
31 changed files with 298 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
net/core/dev.c
View File

@@ -134,7 +134,7 @@
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/net_tstamp.h>
#include <linux/jump_label.h>
#include <linux/static_key.h>
#include <net/flow_keys.h>
#include "net-sysfs.h"
@@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
}
EXPORT_SYMBOL(call_netdevice_notifiers);
static struct jump_label_key netstamp_needed __read_mostly;
static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
/* We are not allowed to call jump_label_dec() from irq context
/* We are not allowed to call static_key_slow_dec() from irq context
* If net_disable_timestamp() is called from irq context, defer the
* jump_label_dec() calls.
* static_key_slow_dec() calls.
*/
static atomic_t netstamp_needed_deferred;
#endif
@@ -1457,12 +1457,12 @@ void net_enable_timestamp(void)
if (deferred) {
while (--deferred)
jump_label_dec(&netstamp_needed);
static_key_slow_dec(&netstamp_needed);
return;
}
#endif
WARN_ON(in_interrupt());
jump_label_inc(&netstamp_needed);
static_key_slow_inc(&netstamp_needed);
}
EXPORT_SYMBOL(net_enable_timestamp);
@@ -1474,19 +1474,19 @@ void net_disable_timestamp(void)
return;
}
#endif
jump_label_dec(&netstamp_needed);
static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp.tv64 = 0;
if (static_branch(&netstamp_needed))
if (static_key_false(&netstamp_needed))
__net_timestamp(skb);
}
#define net_timestamp_check(COND, SKB) \
if (static_branch(&netstamp_needed)) { \
if (static_key_false(&netstamp_needed)) { \
if ((COND) && !(SKB)->tstamp.tv64) \
__net_timestamp(SKB); \
} \
@@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table);
struct jump_label_key rps_needed __read_mostly;
struct static_key rps_needed __read_mostly;
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
@@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb);
#ifdef CONFIG_RPS
if (static_branch(&rps_needed)) {
if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
@@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS;
#ifdef CONFIG_RPS
if (static_branch(&rps_needed)) {
if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;

4
net/core/net-sysfs.c
View File

@@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
spin_unlock(&rps_map_lock);
if (map)
jump_label_inc(&rps_needed);
static_key_slow_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu);
jump_label_dec(&rps_needed);
static_key_slow_dec(&rps_needed);
}
free_cpumask_var(mask);
return len;

4
net/core/sock.c
View File

@@ -111,7 +111,7 @@
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/user_namespace.h>
#include <linux/jump_label.h>
#include <linux/static_key.h>
#include <linux/memcontrol.h>
#include <asm/uaccess.h>
@@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct cgroup *cgrp, struct cgroup_subsys *ss)
static struct lock_class_key af_family_keys[AF_MAX];
static struct lock_class_key af_family_slock_keys[AF_MAX];
struct jump_label_key memcg_socket_limit_enabled;
struct static_key memcg_socket_limit_enabled;
EXPORT_SYMBOL(memcg_socket_limit_enabled);
/*

4
net/core/sysctl_net_core.c
View File

@@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
jump_label_inc(&rps_needed);
static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
jump_label_dec(&rps_needed);
static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}