forked from Minki/linux
e0f6dec35f
In checkin:
0c44c2d0f4 x86: Use asm goto to implement better modify_and_test() functions
the various functions which do modify and test were unified and
optimized using "asm goto". However, this change missed the detail
that the bitops require an "Ir" constraint rather than an "er"
constraint ("I" = integer constant from 0-31, "e" = signed 32-bit
integer constant). This would cause code to miscompile if these
functions were used on constant bit positions 32-255 and the build to
fail if used on constant bit positions above 255.
Add the constraints as a parameter to the GEN_BINARY_RMWcc() macro to
avoid this problem.
Reported-by: Jesse Brandeburg <jesse.brandeburg@intel.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/529E8719.4070202@zytor.com
259 lines
5.8 KiB
C
259 lines
5.8 KiB
C
#ifndef _ASM_X86_ATOMIC_H
|
|
#define _ASM_X86_ATOMIC_H
|
|
|
|
#include <linux/compiler.h>
|
|
#include <linux/types.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/alternative.h>
|
|
#include <asm/cmpxchg.h>
|
|
#include <asm/rmwcc.h>
|
|
|
|
/*
|
|
* Atomic operations that C can't guarantee us. Useful for
|
|
* resource counting etc..
|
|
*/
|
|
|
|
#define ATOMIC_INIT(i) { (i) }
|
|
|
|
/**
|
|
* atomic_read - read atomic variable
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically reads the value of @v.
|
|
*/
|
|
static inline int atomic_read(const atomic_t *v)
|
|
{
|
|
return (*(volatile int *)&(v)->counter);
|
|
}
|
|
|
|
/**
|
|
* atomic_set - set atomic variable
|
|
* @v: pointer of type atomic_t
|
|
* @i: required value
|
|
*
|
|
* Atomically sets the value of @v to @i.
|
|
*/
|
|
static inline void atomic_set(atomic_t *v, int i)
|
|
{
|
|
v->counter = i;
|
|
}
|
|
|
|
/**
|
|
* atomic_add - add integer to atomic variable
|
|
* @i: integer value to add
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically adds @i to @v.
|
|
*/
|
|
static inline void atomic_add(int i, atomic_t *v)
|
|
{
|
|
asm volatile(LOCK_PREFIX "addl %1,%0"
|
|
: "+m" (v->counter)
|
|
: "ir" (i));
|
|
}
|
|
|
|
/**
|
|
* atomic_sub - subtract integer from atomic variable
|
|
* @i: integer value to subtract
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically subtracts @i from @v.
|
|
*/
|
|
static inline void atomic_sub(int i, atomic_t *v)
|
|
{
|
|
asm volatile(LOCK_PREFIX "subl %1,%0"
|
|
: "+m" (v->counter)
|
|
: "ir" (i));
|
|
}
|
|
|
|
/**
|
|
* atomic_sub_and_test - subtract value from variable and test result
|
|
* @i: integer value to subtract
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically subtracts @i from @v and returns
|
|
* true if the result is zero, or false for all
|
|
* other cases.
|
|
*/
|
|
static inline int atomic_sub_and_test(int i, atomic_t *v)
|
|
{
|
|
GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", "e");
|
|
}
|
|
|
|
/**
|
|
* atomic_inc - increment atomic variable
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically increments @v by 1.
|
|
*/
|
|
static inline void atomic_inc(atomic_t *v)
|
|
{
|
|
asm volatile(LOCK_PREFIX "incl %0"
|
|
: "+m" (v->counter));
|
|
}
|
|
|
|
/**
|
|
* atomic_dec - decrement atomic variable
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically decrements @v by 1.
|
|
*/
|
|
static inline void atomic_dec(atomic_t *v)
|
|
{
|
|
asm volatile(LOCK_PREFIX "decl %0"
|
|
: "+m" (v->counter));
|
|
}
|
|
|
|
/**
|
|
* atomic_dec_and_test - decrement and test
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically decrements @v by 1 and
|
|
* returns true if the result is 0, or false for all other
|
|
* cases.
|
|
*/
|
|
static inline int atomic_dec_and_test(atomic_t *v)
|
|
{
|
|
GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", "e");
|
|
}
|
|
|
|
/**
|
|
* atomic_inc_and_test - increment and test
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically increments @v by 1
|
|
* and returns true if the result is zero, or false for all
|
|
* other cases.
|
|
*/
|
|
static inline int atomic_inc_and_test(atomic_t *v)
|
|
{
|
|
GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", "e");
|
|
}
|
|
|
|
/**
|
|
* atomic_add_negative - add and test if negative
|
|
* @i: integer value to add
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically adds @i to @v and returns true
|
|
* if the result is negative, or false when
|
|
* result is greater than or equal to zero.
|
|
*/
|
|
static inline int atomic_add_negative(int i, atomic_t *v)
|
|
{
|
|
GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", "s");
|
|
}
|
|
|
|
/**
|
|
* atomic_add_return - add integer and return
|
|
* @i: integer value to add
|
|
* @v: pointer of type atomic_t
|
|
*
|
|
* Atomically adds @i to @v and returns @i + @v
|
|
*/
|
|
static inline int atomic_add_return(int i, atomic_t *v)
|
|
{
|
|
return i + xadd(&v->counter, i);
|
|
}
|
|
|
|
/**
|
|
* atomic_sub_return - subtract integer and return
|
|
* @v: pointer of type atomic_t
|
|
* @i: integer value to subtract
|
|
*
|
|
* Atomically subtracts @i from @v and returns @v - @i
|
|
*/
|
|
static inline int atomic_sub_return(int i, atomic_t *v)
|
|
{
|
|
return atomic_add_return(-i, v);
|
|
}
|
|
|
|
#define atomic_inc_return(v) (atomic_add_return(1, v))
|
|
#define atomic_dec_return(v) (atomic_sub_return(1, v))
|
|
|
|
static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
|
|
{
|
|
return cmpxchg(&v->counter, old, new);
|
|
}
|
|
|
|
static inline int atomic_xchg(atomic_t *v, int new)
|
|
{
|
|
return xchg(&v->counter, new);
|
|
}
|
|
|
|
/**
|
|
* __atomic_add_unless - add unless the number is already a given value
|
|
* @v: pointer of type atomic_t
|
|
* @a: the amount to add to v...
|
|
* @u: ...unless v is equal to u.
|
|
*
|
|
* Atomically adds @a to @v, so long as @v was not already @u.
|
|
* Returns the old value of @v.
|
|
*/
|
|
static inline int __atomic_add_unless(atomic_t *v, int a, int u)
|
|
{
|
|
int c, old;
|
|
c = atomic_read(v);
|
|
for (;;) {
|
|
if (unlikely(c == (u)))
|
|
break;
|
|
old = atomic_cmpxchg((v), c, c + (a));
|
|
if (likely(old == c))
|
|
break;
|
|
c = old;
|
|
}
|
|
return c;
|
|
}
|
|
|
|
/**
|
|
* atomic_inc_short - increment of a short integer
|
|
* @v: pointer to type int
|
|
*
|
|
* Atomically adds 1 to @v
|
|
* Returns the new value of @u
|
|
*/
|
|
static inline short int atomic_inc_short(short int *v)
|
|
{
|
|
asm(LOCK_PREFIX "addw $1, %0" : "+m" (*v));
|
|
return *v;
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
/**
|
|
* atomic_or_long - OR of two long integers
|
|
* @v1: pointer to type unsigned long
|
|
* @v2: pointer to type unsigned long
|
|
*
|
|
* Atomically ORs @v1 and @v2
|
|
* Returns the result of the OR
|
|
*/
|
|
static inline void atomic_or_long(unsigned long *v1, unsigned long v2)
|
|
{
|
|
asm(LOCK_PREFIX "orq %1, %0" : "+m" (*v1) : "r" (v2));
|
|
}
|
|
#endif
|
|
|
|
/* These are x86-specific, used by some header files */
|
|
#define atomic_clear_mask(mask, addr) \
|
|
asm volatile(LOCK_PREFIX "andl %0,%1" \
|
|
: : "r" (~(mask)), "m" (*(addr)) : "memory")
|
|
|
|
#define atomic_set_mask(mask, addr) \
|
|
asm volatile(LOCK_PREFIX "orl %0,%1" \
|
|
: : "r" ((unsigned)(mask)), "m" (*(addr)) \
|
|
: "memory")
|
|
|
|
/* Atomic operations are already serializing on x86 */
|
|
#define smp_mb__before_atomic_dec() barrier()
|
|
#define smp_mb__after_atomic_dec() barrier()
|
|
#define smp_mb__before_atomic_inc() barrier()
|
|
#define smp_mb__after_atomic_inc() barrier()
|
|
|
|
#ifdef CONFIG_X86_32
|
|
# include <asm/atomic64_32.h>
|
|
#else
|
|
# include <asm/atomic64_64.h>
|
|
#endif
|
|
|
|
#endif /* _ASM_X86_ATOMIC_H */
|