47933ad41a
A number of situations currently require the heavyweight smp_mb(), even though there is no need to order prior stores against later loads. Many architectures have much cheaper ways to handle these situations, but the Linux kernel currently has no portable way to make use of them. This commit therefore supplies smp_load_acquire() and smp_store_release() to remedy this situation. The new smp_load_acquire() primitive orders the specified load against any subsequent reads or writes, while the new smp_store_release() primitive orders the specifed store against any prior reads or writes. These primitives allow array-based circular FIFOs to be implemented without an smp_mb(), and also allow a theoretical hole in rcu_assign_pointer() to be closed at no additional expense on most architectures. In addition, the RCU experience transitioning from explicit smp_read_barrier_depends() and smp_wmb() to rcu_dereference() and rcu_assign_pointer(), respectively resulted in substantial improvements in readability. It therefore seems likely that replacing other explicit barriers with smp_load_acquire() and smp_store_release() will provide similar benefits. It appears that roughly half of the explicit barriers in core kernel code might be so replaced. [Changelog by PaulMck] Reviewed-by: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Signed-off-by: Peter Zijlstra <peterz@infradead.org> Acked-by: Will Deacon <will.deacon@arm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> Cc: Michael Ellerman <michael@ellerman.id.au> Cc: Michael Neuling <mikey@neuling.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Geert Uytterhoeven <geert@linux-m68k.org> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Victor Kaplansky <VICTORK@il.ibm.com> Cc: Tony Luck <tony.luck@intel.com> Cc: Oleg Nesterov <oleg@redhat.com> Link: http://lkml.kernel.org/r/20131213150640.908486364@infradead.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
88 lines
2.9 KiB
C
88 lines
2.9 KiB
C
/*
|
|
* Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
|
|
*/
|
|
#ifndef _ASM_POWERPC_BARRIER_H
|
|
#define _ASM_POWERPC_BARRIER_H
|
|
|
|
/*
|
|
* Memory barrier.
|
|
* The sync instruction guarantees that all memory accesses initiated
|
|
* by this processor have been performed (with respect to all other
|
|
* mechanisms that access memory). The eieio instruction is a barrier
|
|
* providing an ordering (separately) for (a) cacheable stores and (b)
|
|
* loads and stores to non-cacheable memory (e.g. I/O devices).
|
|
*
|
|
* mb() prevents loads and stores being reordered across this point.
|
|
* rmb() prevents loads being reordered across this point.
|
|
* wmb() prevents stores being reordered across this point.
|
|
* read_barrier_depends() prevents data-dependent loads being reordered
|
|
* across this point (nop on PPC).
|
|
*
|
|
* *mb() variants without smp_ prefix must order all types of memory
|
|
* operations with one another. sync is the only instruction sufficient
|
|
* to do this.
|
|
*
|
|
* For the smp_ barriers, ordering is for cacheable memory operations
|
|
* only. We have to use the sync instruction for smp_mb(), since lwsync
|
|
* doesn't order loads with respect to previous stores. Lwsync can be
|
|
* used for smp_rmb() and smp_wmb().
|
|
*
|
|
* However, on CPUs that don't support lwsync, lwsync actually maps to a
|
|
* heavy-weight sync, so smp_wmb() can be a lighter-weight eieio.
|
|
*/
|
|
#define mb() __asm__ __volatile__ ("sync" : : : "memory")
|
|
#define rmb() __asm__ __volatile__ ("sync" : : : "memory")
|
|
#define wmb() __asm__ __volatile__ ("sync" : : : "memory")
|
|
#define read_barrier_depends() do { } while(0)
|
|
|
|
#define set_mb(var, value) do { var = value; mb(); } while (0)
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
#ifdef __SUBARCH_HAS_LWSYNC
|
|
# define SMPWMB LWSYNC
|
|
#else
|
|
# define SMPWMB eieio
|
|
#endif
|
|
|
|
#define __lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory")
|
|
|
|
#define smp_mb() mb()
|
|
#define smp_rmb() __lwsync()
|
|
#define smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory")
|
|
#define smp_read_barrier_depends() read_barrier_depends()
|
|
#else
|
|
#define __lwsync() barrier()
|
|
|
|
#define smp_mb() barrier()
|
|
#define smp_rmb() barrier()
|
|
#define smp_wmb() barrier()
|
|
#define smp_read_barrier_depends() do { } while(0)
|
|
#endif /* CONFIG_SMP */
|
|
|
|
/*
|
|
* This is a barrier which prevents following instructions from being
|
|
* started until the value of the argument x is known. For example, if
|
|
* x is a variable loaded from memory, this prevents following
|
|
* instructions from being executed until the load has been performed.
|
|
*/
|
|
#define data_barrier(x) \
|
|
asm volatile("twi 0,%0,0; isync" : : "r" (x) : "memory");
|
|
|
|
#define smp_store_release(p, v) \
|
|
do { \
|
|
compiletime_assert_atomic_type(*p); \
|
|
__lwsync(); \
|
|
ACCESS_ONCE(*p) = (v); \
|
|
} while (0)
|
|
|
|
#define smp_load_acquire(p) \
|
|
({ \
|
|
typeof(*p) ___p1 = ACCESS_ONCE(*p); \
|
|
compiletime_assert_atomic_type(*p); \
|
|
__lwsync(); \
|
|
___p1; \
|
|
})
|
|
|
|
#endif /* _ASM_POWERPC_BARRIER_H */
|