mirror of
https://github.com/torvalds/linux.git
synced 2024-11-06 03:51:48 +00:00
11276d5306
There are various problems and short-comings with the current
static_key interface:
- static_key_{true,false}() read like a branch depending on the key
value, instead of the actual likely/unlikely branch depending on
init value.
- static_key_{true,false}() are, as stated above, tied to the
static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.
- we're limited to the 2 (out of 4) possible options that compile to
a default NOP because that's what our arch_static_branch() assembly
emits.
So provide a new static_key interface:
DEFINE_STATIC_KEY_TRUE(name);
DEFINE_STATIC_KEY_FALSE(name);
Which define a key of different types with an initial true/false
value.
Then allow:
static_branch_likely()
static_branch_unlikely()
to take a key of either type and emit the right instruction for the
case.
This means adding a second arch_static_branch_jump() assembly helper
which emits a JMP per default.
In order to determine the right instruction for the right state,
encode the branch type in the LSB of jump_entry::key.
This is the final step in removing the naming confusion that has led to
a stream of avoidable bugs such as:
a833581e37 ("x86, perf: Fix static_key bug in load_mm_cr4()")
... but it also allows new static key combinations that will give us
performance enhancements in the subsequent patches.
Tested-by: Rabin Vincent <rabin@rab.in> # arm
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> # ppc
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com> # s390
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
70 lines
1.5 KiB
C
70 lines
1.5 KiB
C
#ifndef _ASM_POWERPC_JUMP_LABEL_H
|
|
#define _ASM_POWERPC_JUMP_LABEL_H
|
|
|
|
/*
|
|
* Copyright 2010 Michael Ellerman, IBM Corp.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*/
|
|
|
|
#ifndef __ASSEMBLY__
|
|
#include <linux/types.h>
|
|
|
|
#include <asm/feature-fixups.h>
|
|
|
|
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
|
|
#define JUMP_LABEL_NOP_SIZE 4
|
|
|
|
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("1:\n\t"
|
|
"nop\n\t"
|
|
".pushsection __jump_table, \"aw\"\n\t"
|
|
JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
|
|
".popsection \n\t"
|
|
: : "i" (&((char *)key)[branch]) : : l_yes);
|
|
|
|
return false;
|
|
l_yes:
|
|
return true;
|
|
}
|
|
|
|
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("1:\n\t"
|
|
"b %l[l_yes]\n\t"
|
|
".pushsection __jump_table, \"aw\"\n\t"
|
|
JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
|
|
".popsection \n\t"
|
|
: : "i" (&((char *)key)[branch]) : : l_yes);
|
|
|
|
return false;
|
|
l_yes:
|
|
return true;
|
|
}
|
|
|
|
#ifdef CONFIG_PPC64
|
|
typedef u64 jump_label_t;
|
|
#else
|
|
typedef u32 jump_label_t;
|
|
#endif
|
|
|
|
struct jump_entry {
|
|
jump_label_t code;
|
|
jump_label_t target;
|
|
jump_label_t key;
|
|
};
|
|
|
|
#else
|
|
#define ARCH_STATIC_BRANCH(LABEL, KEY) \
|
|
1098: nop; \
|
|
.pushsection __jump_table, "aw"; \
|
|
FTR_ENTRY_LONG 1098b, LABEL, KEY; \
|
|
.popsection
|
|
#endif
|
|
|
|
#endif /* _ASM_POWERPC_JUMP_LABEL_H */
|