mirror of
https://github.com/torvalds/linux.git
synced 2024-11-23 04:31:50 +00:00
9a07731702
test_barrier fails on s390 because of the missing KCSAN instrumentation for several synchronization primitives. Add it to barriers by defining __mb(), __rmb(), __wmb(), __dma_rmb() and __dma_wmb(), and letting the common code in asm-generic/barrier.h do the rest. Spinlocks require instrumentation only on the unlock path; notify KCSAN that the CPU cannot move memory accesses outside of the spin lock. In reality it also cannot move stores inside of it, but this is not important and can be omitted. Reported-by: Tobias Huschle <huschle@linux.ibm.com> Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
83 lines
1.9 KiB
C
83 lines
1.9 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright IBM Corp. 1999, 2009
|
|
*
|
|
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
|
|
*/
|
|
|
|
#ifndef __ASM_BARRIER_H
|
|
#define __ASM_BARRIER_H
|
|
|
|
/*
|
|
* Force strict CPU ordering.
|
|
* And yes, this is required on UP too when we're talking
|
|
* to devices.
|
|
*/
|
|
|
|
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
|
|
/* Fast-BCR without checkpoint synchronization */
|
|
#define __ASM_BCR_SERIALIZE "bcr 14,0\n"
|
|
#else
|
|
#define __ASM_BCR_SERIALIZE "bcr 15,0\n"
|
|
#endif
|
|
|
|
static __always_inline void bcr_serialize(void)
|
|
{
|
|
asm volatile(__ASM_BCR_SERIALIZE : : : "memory");
|
|
}
|
|
|
|
#define __mb() bcr_serialize()
|
|
#define __rmb() barrier()
|
|
#define __wmb() barrier()
|
|
#define __dma_rmb() __mb()
|
|
#define __dma_wmb() __mb()
|
|
#define __smp_mb() __mb()
|
|
#define __smp_rmb() __rmb()
|
|
#define __smp_wmb() __wmb()
|
|
|
|
#define __smp_store_release(p, v) \
|
|
do { \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
WRITE_ONCE(*p, v); \
|
|
} while (0)
|
|
|
|
#define __smp_load_acquire(p) \
|
|
({ \
|
|
typeof(*p) ___p1 = READ_ONCE(*p); \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
___p1; \
|
|
})
|
|
|
|
#define __smp_mb__before_atomic() barrier()
|
|
#define __smp_mb__after_atomic() barrier()
|
|
|
|
/**
|
|
* array_index_mask_nospec - generate a mask for array_idx() that is
|
|
* ~0UL when the bounds check succeeds and 0 otherwise
|
|
* @index: array element index
|
|
* @size: number of elements in array
|
|
*/
|
|
#define array_index_mask_nospec array_index_mask_nospec
|
|
static inline unsigned long array_index_mask_nospec(unsigned long index,
|
|
unsigned long size)
|
|
{
|
|
unsigned long mask;
|
|
|
|
if (__builtin_constant_p(size) && size > 0) {
|
|
asm(" clgr %2,%1\n"
|
|
" slbgr %0,%0\n"
|
|
:"=d" (mask) : "d" (size-1), "d" (index) :"cc");
|
|
return mask;
|
|
}
|
|
asm(" clgr %1,%2\n"
|
|
" slbgr %0,%0\n"
|
|
:"=d" (mask) : "d" (size), "d" (index) :"cc");
|
|
return ~mask;
|
|
}
|
|
|
|
#include <asm-generic/barrier.h>
|
|
|
|
#endif /* __ASM_BARRIER_H */
|