mirror of
https://github.com/torvalds/linux.git
synced 2024-12-24 11:51:27 +00:00
47933ad41a
A number of situations currently require the heavyweight smp_mb(), even though there is no need to order prior stores against later loads. Many architectures have much cheaper ways to handle these situations, but the Linux kernel currently has no portable way to make use of them. This commit therefore supplies smp_load_acquire() and smp_store_release() to remedy this situation. The new smp_load_acquire() primitive orders the specified load against any subsequent reads or writes, while the new smp_store_release() primitive orders the specifed store against any prior reads or writes. These primitives allow array-based circular FIFOs to be implemented without an smp_mb(), and also allow a theoretical hole in rcu_assign_pointer() to be closed at no additional expense on most architectures. In addition, the RCU experience transitioning from explicit smp_read_barrier_depends() and smp_wmb() to rcu_dereference() and rcu_assign_pointer(), respectively resulted in substantial improvements in readability. It therefore seems likely that replacing other explicit barriers with smp_load_acquire() and smp_store_release() will provide similar benefits. It appears that roughly half of the explicit barriers in core kernel code might be so replaced. [Changelog by PaulMck] Reviewed-by: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Signed-off-by: Peter Zijlstra <peterz@infradead.org> Acked-by: Will Deacon <will.deacon@arm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> Cc: Michael Ellerman <michael@ellerman.id.au> Cc: Michael Neuling <mikey@neuling.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Geert Uytterhoeven <geert@linux-m68k.org> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Victor Kaplansky <VICTORK@il.ibm.com> Cc: Tony Luck <tony.luck@intel.com> Cc: Oleg Nesterov <oleg@redhat.com> Link: http://lkml.kernel.org/r/20131213150640.908486364@infradead.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
72 lines
2.2 KiB
C
72 lines
2.2 KiB
C
#ifndef __SPARC64_BARRIER_H
|
|
#define __SPARC64_BARRIER_H
|
|
|
|
/* These are here in an effort to more fully work around Spitfire Errata
|
|
* #51. Essentially, if a memory barrier occurs soon after a mispredicted
|
|
* branch, the chip can stop executing instructions until a trap occurs.
|
|
* Therefore, if interrupts are disabled, the chip can hang forever.
|
|
*
|
|
* It used to be believed that the memory barrier had to be right in the
|
|
* delay slot, but a case has been traced recently wherein the memory barrier
|
|
* was one instruction after the branch delay slot and the chip still hung.
|
|
* The offending sequence was the following in sym_wakeup_done() of the
|
|
* sym53c8xx_2 driver:
|
|
*
|
|
* call sym_ccb_from_dsa, 0
|
|
* movge %icc, 0, %l0
|
|
* brz,pn %o0, .LL1303
|
|
* mov %o0, %l2
|
|
* membar #LoadLoad
|
|
*
|
|
* The branch has to be mispredicted for the bug to occur. Therefore, we put
|
|
* the memory barrier explicitly into a "branch always, predicted taken"
|
|
* delay slot to avoid the problem case.
|
|
*/
|
|
#define membar_safe(type) \
|
|
do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \
|
|
" membar " type "\n" \
|
|
"1:\n" \
|
|
: : : "memory"); \
|
|
} while (0)
|
|
|
|
/* The kernel always executes in TSO memory model these days,
|
|
* and furthermore most sparc64 chips implement more stringent
|
|
* memory ordering than required by the specifications.
|
|
*/
|
|
#define mb() membar_safe("#StoreLoad")
|
|
#define rmb() __asm__ __volatile__("":::"memory")
|
|
#define wmb() __asm__ __volatile__("":::"memory")
|
|
|
|
#define read_barrier_depends() do { } while(0)
|
|
#define set_mb(__var, __value) \
|
|
do { __var = __value; membar_safe("#StoreLoad"); } while(0)
|
|
|
|
#ifdef CONFIG_SMP
|
|
#define smp_mb() mb()
|
|
#define smp_rmb() rmb()
|
|
#define smp_wmb() wmb()
|
|
#else
|
|
#define smp_mb() __asm__ __volatile__("":::"memory")
|
|
#define smp_rmb() __asm__ __volatile__("":::"memory")
|
|
#define smp_wmb() __asm__ __volatile__("":::"memory")
|
|
#endif
|
|
|
|
#define smp_read_barrier_depends() do { } while(0)
|
|
|
|
#define smp_store_release(p, v) \
|
|
do { \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
ACCESS_ONCE(*p) = (v); \
|
|
} while (0)
|
|
|
|
#define smp_load_acquire(p) \
|
|
({ \
|
|
typeof(*p) ___p1 = ACCESS_ONCE(*p); \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
___p1; \
|
|
})
|
|
|
|
#endif /* !(__SPARC64_BARRIER_H) */
|