11276d5306
There are various problems and short-comings with the current
static_key interface:
- static_key_{true,false}() read like a branch depending on the key
value, instead of the actual likely/unlikely branch depending on
init value.
- static_key_{true,false}() are, as stated above, tied to the
static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.
- we're limited to the 2 (out of 4) possible options that compile to
a default NOP because that's what our arch_static_branch() assembly
emits.
So provide a new static_key interface:
DEFINE_STATIC_KEY_TRUE(name);
DEFINE_STATIC_KEY_FALSE(name);
Which define a key of different types with an initial true/false
value.
Then allow:
static_branch_likely()
static_branch_unlikely()
to take a key of either type and emit the right instruction for the
case.
This means adding a second arch_static_branch_jump() assembly helper
which emits a JMP per default.
In order to determine the right instruction for the right state,
encode the branch type in the LSB of jump_entry::key.
This is the final step in removing the naming confusion that has led to
a stream of avoidable bugs such as:
a833581e37 ("x86, perf: Fix static_key bug in load_mm_cr4()")
... but it also allows new static key combinations that will give us
performance enhancements in the subsequent patches.
Tested-by: Rabin Vincent <rabin@rab.in> # arm
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> # ppc
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com> # s390
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
53 lines
1.1 KiB
C
53 lines
1.1 KiB
C
#ifndef _ASM_S390_JUMP_LABEL_H
|
|
#define _ASM_S390_JUMP_LABEL_H
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/types.h>
|
|
|
|
#define JUMP_LABEL_NOP_SIZE 6
|
|
#define JUMP_LABEL_NOP_OFFSET 2
|
|
|
|
/*
|
|
* We use a brcl 0,2 instruction for jump labels at compile time so it
|
|
* can be easily distinguished from a hotpatch generated instruction.
|
|
*/
|
|
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
|
|
".pushsection __jump_table, \"aw\"\n"
|
|
".balign 8\n"
|
|
".quad 0b, %l[label], %0\n"
|
|
".popsection\n"
|
|
: : "X" (&((char *)key)[branch]) : : label);
|
|
|
|
return false;
|
|
label:
|
|
return true;
|
|
}
|
|
|
|
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("0: brcl 15, %l[label]\n"
|
|
".pushsection __jump_table, \"aw\"\n"
|
|
".balign 8\n"
|
|
".quad 0b, %l[label], %0\n"
|
|
".popsection\n"
|
|
: : "X" (&((char *)key)[branch]) : : label);
|
|
|
|
return false;
|
|
label:
|
|
return true;
|
|
}
|
|
|
|
typedef unsigned long jump_label_t;
|
|
|
|
struct jump_entry {
|
|
jump_label_t code;
|
|
jump_label_t target;
|
|
jump_label_t key;
|
|
};
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
#endif
|