leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

arm64: Rewrite __arch_clear_user()

Browse Source 
Now that we're always using STTR variants rather than abstracting two
different addressing modes, the user_ldst macro here is frankly more
obfuscating than helpful. Rewrite __arch_clear_user() with regular
USER() annotations so that it's clearer what's going on, and take the
opportunity to minimise the branchiness in the most common paths, while
also allowing the exception fixup to return an accurate result.

Apparently some folks examine large reads from /dev/zero closely enough
to notice the loop being hot, so align it per the other critical loops
(presumably around a typical instruction fetch granularity).

Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Link: https://lore.kernel.org/r/1cbd78b12c076a8ad4656a345811cfb9425df0b3.1622128527.git.robin.murphy@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Robin Murphy 2021-05-27 16:34:48 +01:00 committed by Will Deacon
parent 9e51cafd78
commit 344323e042
1 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
arch/arm64/lib/clear_user.S
View File

@ -1,12 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Based on arch/arm/lib/clear_user.S
*
* Copyright (C) 2012 ARM Ltd.
* Copyright (C) 2021 Arm Ltd.
*/
#include <linux/linkage.h>
#include <asm/asm-uaccess.h>
#include <linux/linkage.h>
#include <asm/assembler.h>
.text
@ -19,25 +16,33 @@
*
* Alignment fixed up by hardware.
*/
.p2align 4
// Alignment is for the loop, but since the prologue (including BTI)
// is also 16 bytes we can keep any padding outside the function
SYM_FUNC_START(__arch_clear_user)
mov x2, x1 // save the size for fixup return
add x2, x0, x1
subs x1, x1, #8
b.mi 2f
1:
user_ldst 9f, sttr, xzr, x0, 8
USER(9f, sttr xzr, [x0])
add x0, x0, #8
subs x1, x1, #8
b.pl 1b
2: adds x1, x1, #4
b.mi 3f
user_ldst 9f, sttr, wzr, x0, 4
sub x1, x1, #4
3: adds x1, x1, #2
b.mi 4f
user_ldst 9f, sttrh, wzr, x0, 2
sub x1, x1, #2
4: adds x1, x1, #1
b.mi 5f
user_ldst 9f, sttrb, wzr, x0, 0
b.hi 1b
USER(9f, sttr xzr, [x2, #-8])
mov x0, #0
ret
2: tbz x1, #2, 3f
USER(9f, sttr wzr, [x0])
USER(8f, sttr wzr, [x2, #-4])
mov x0, #0
ret
3: tbz x1, #1, 4f
USER(9f, sttrh wzr, [x0])
4: tbz x1, #0, 5f
USER(7f, sttrb wzr, [x2, #-1])
5: mov x0, #0
ret
SYM_FUNC_END(__arch_clear_user)
@ -45,6 +50,8 @@ EXPORT_SYMBOL(__arch_clear_user)
.section .fixup,"ax"
.align 2
9: mov x0, x2 // return the original size
7: sub x0, x2, #5 // Adjust for faulting on the final byte...
8: add x0, x0, #4 // ...or the second word of the 4-7 byte case
9: sub x0, x2, x0
ret
.previous