mirror of
https://github.com/torvalds/linux.git
synced 2024-12-04 18:13:04 +00:00
c4e34dd99f
The exception for the "unaligned access at the end of the page, next page not mapped" never happens, but the fixup code ends up causing trouble for compilers to optimize well. clang in particular ends up seeing it being in the middle of a loop, and tries desperately to optimize the exception fixup code that is never really reached. The simple solution is to just move all the fixups into the exception handler itself, which moves it all out of the hot case code, and means that the compiler never sees it or needs to worry about it. Acked-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
95 lines
2.3 KiB
C
95 lines
2.3 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_WORD_AT_A_TIME_H
|
|
#define _ASM_WORD_AT_A_TIME_H
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
/*
|
|
* This is largely generic for little-endian machines, but the
|
|
* optimal byte mask counting is probably going to be something
|
|
* that is architecture-specific. If you have a reliably fast
|
|
* bit count instruction, that might be better than the multiply
|
|
* and shift, for example.
|
|
*/
|
|
struct word_at_a_time {
|
|
const unsigned long one_bits, high_bits;
|
|
};
|
|
|
|
#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
|
|
|
|
#ifdef CONFIG_64BIT
|
|
|
|
/*
|
|
* Jan Achrenius on G+: microoptimized version of
|
|
* the simpler "(mask & ONEBYTES) * ONEBYTES >> 56"
|
|
* that works for the bytemasks without having to
|
|
* mask them first.
|
|
*/
|
|
static inline long count_masked_bytes(unsigned long mask)
|
|
{
|
|
return mask*0x0001020304050608ul >> 56;
|
|
}
|
|
|
|
#else /* 32-bit case */
|
|
|
|
/* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
|
|
static inline long count_masked_bytes(long mask)
|
|
{
|
|
/* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
|
|
long a = (0x0ff0001+mask) >> 23;
|
|
/* Fix the 1 for 00 case */
|
|
return a & mask;
|
|
}
|
|
|
|
#endif
|
|
|
|
/* Return nonzero if it has a zero */
|
|
static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
|
|
{
|
|
unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
|
|
*bits = mask;
|
|
return mask;
|
|
}
|
|
|
|
static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
|
|
{
|
|
return bits;
|
|
}
|
|
|
|
static inline unsigned long create_zero_mask(unsigned long bits)
|
|
{
|
|
bits = (bits - 1) & ~bits;
|
|
return bits >> 7;
|
|
}
|
|
|
|
/* The mask we created is directly usable as a bytemask */
|
|
#define zero_bytemask(mask) (mask)
|
|
|
|
static inline unsigned long find_zero(unsigned long mask)
|
|
{
|
|
return count_masked_bytes(mask);
|
|
}
|
|
|
|
/*
|
|
* Load an unaligned word from kernel space.
|
|
*
|
|
* In the (very unlikely) case of the word being a page-crosser
|
|
* and the next page not being mapped, take the exception and
|
|
* return zeroes in the non-existing part.
|
|
*/
|
|
static inline unsigned long load_unaligned_zeropad(const void *addr)
|
|
{
|
|
unsigned long ret;
|
|
|
|
asm volatile(
|
|
"1: mov %[mem], %[ret]\n"
|
|
"2:\n"
|
|
_ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_ZEROPAD)
|
|
: [ret] "=r" (ret)
|
|
: [mem] "m" (*(unsigned long *)addr));
|
|
|
|
return ret;
|
|
}
|
|
|
|
#endif /* _ASM_WORD_AT_A_TIME_H */
|