forked from Minki/linux
4423028203
Use a memory barrier + store sequence instead of a load + compare and swap sequence to unlock a spinlock and an rw lock. For the spinlock case this saves us two memory reads and a not needed cpu serialization after the compare and swap instruction stored the new value. The kernel size (performance_defconfig) gets reduced by ~14k. Average execution time of a tight inlined spin_unlock loop drops from 5.8ns to 0.7ns on a zEC12 machine. An artificial stress test case where several counters are protected with a single spinlock and which are only incremented while holding the spinlock shows ~30% improvement on a 4 cpu machine. Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
54 lines
1.2 KiB
C
54 lines
1.2 KiB
C
/*
|
|
* Copyright IBM Corp. 1999, 2009
|
|
*
|
|
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
|
|
*/
|
|
|
|
#ifndef __ASM_BARRIER_H
|
|
#define __ASM_BARRIER_H
|
|
|
|
/*
|
|
* Force strict CPU ordering.
|
|
* And yes, this is required on UP too when we're talking
|
|
* to devices.
|
|
*/
|
|
|
|
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
|
|
/* Fast-BCR without checkpoint synchronization */
|
|
#define __ASM_BARRIER "bcr 14,0\n"
|
|
#else
|
|
#define __ASM_BARRIER "bcr 15,0\n"
|
|
#endif
|
|
|
|
#define mb() do { asm volatile(__ASM_BARRIER : : : "memory"); } while (0)
|
|
|
|
#define rmb() mb()
|
|
#define wmb() mb()
|
|
#define read_barrier_depends() do { } while(0)
|
|
#define smp_mb() mb()
|
|
#define smp_rmb() rmb()
|
|
#define smp_wmb() wmb()
|
|
#define smp_read_barrier_depends() read_barrier_depends()
|
|
|
|
#define smp_mb__before_atomic() smp_mb()
|
|
#define smp_mb__after_atomic() smp_mb()
|
|
|
|
#define set_mb(var, value) do { var = value; mb(); } while (0)
|
|
|
|
#define smp_store_release(p, v) \
|
|
do { \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
ACCESS_ONCE(*p) = (v); \
|
|
} while (0)
|
|
|
|
#define smp_load_acquire(p) \
|
|
({ \
|
|
typeof(*p) ___p1 = ACCESS_ONCE(*p); \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
___p1; \
|
|
})
|
|
|
|
#endif /* __ASM_BARRIER_H */
|