From bd52cd5e23f134019b23f0c389db0f9a436e4576 Mon Sep 17 00:00:00 2001
From: Jann Horn <jannh@google.com>
Date: Tue, 17 May 2022 16:30:47 +0200
Subject: [PATCH 01/17] s390/crypto: fix scatterwalk_unmap() callers in AES-GCM
The argument of scatterwalk_unmap() is supposed to be the void* that was
returned by the previous scatterwalk_map() call.
The s390 AES-GCM implementation was instead passing the pointer to the
struct scatter_walk.
This doesn't actually break anything because scatterwalk_unmap() only uses
its argument under CONFIG_HIGHMEM and ARCH_HAS_FLUSH_ON_KUNMAP.
Fixes: bf7fa038707c ("s390/crypto: add s390 platform specific aes gcm support.")
Signed-off-by: Jann Horn <jannh@google.com>
Acked-by: Harald Freudenberger <freude@linux.ibm.com>
Link: https://lore.kernel.org/r/20220517143047.3054498-1-jannh@google.com
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/crypto/aes_s390.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c
index 54c7536f2482..1023e9d43d44 100644
--- a/arch/s390/crypto/aes_s390.c
+++ b/arch/s390/crypto/aes_s390.c
@@ -701,7 +701,7 @@ static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw,
unsigned int nbytes)
{
gw->walk_bytes_remain -= nbytes;
- scatterwalk_unmap(&gw->walk);
+ scatterwalk_unmap(gw->walk_ptr);
scatterwalk_advance(&gw->walk, nbytes);
scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain);
gw->walk_ptr = NULL;
@@ -776,7 +776,7 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)
goto out;
}
- scatterwalk_unmap(&gw->walk);
+ scatterwalk_unmap(gw->walk_ptr);
gw->walk_ptr = NULL;
gw->ptr = gw->buf;
From 85806016acb0610e47ffcd5415d1575f18f58a33 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Fri, 20 May 2022 19:23:48 +0200
Subject: [PATCH 02/17] s390: simplify early program check handler
Due to historic reasons the base program check handler calls a
configurable function. Given that there is only the early program
check handler left, simplify the code by directly calling that
function.
The only other user was removed with commit d485235b0054 ("s390:
assume diag308 set always works").
Also rename all functions and the asm file to reflect this.
Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/processor.h | 6 -----
arch/s390/kernel/Makefile | 2 +-
arch/s390/kernel/early.c | 5 ++---
arch/s390/kernel/{base.S => earlypgm.S} | 30 +++----------------------
arch/s390/kernel/entry.h | 2 ++
5 files changed, 8 insertions(+), 37 deletions(-)
rename arch/s390/kernel/{base.S => earlypgm.S} (55%)
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index add764a2be8c..bd66f8e34949 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -304,12 +304,6 @@ static __always_inline void __noreturn disabled_wait(void)
while (1);
}
-/*
- * Basic Program Check Handler.
- */
-extern void s390_base_pgm_handler(void);
-extern void (*s390_base_pgm_handler_fn)(struct pt_regs *regs);
-
#define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL
extern int memcpy_real(void *, unsigned long, size_t);
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index 5851041bb214..27d6b3c7aa06 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -33,7 +33,7 @@ CFLAGS_stacktrace.o += -fno-optimize-sibling-calls
CFLAGS_dumpstack.o += -fno-optimize-sibling-calls
CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls
-obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o
+obj-y := traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o
obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o
obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o
obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o
diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 08cc86a0db90..432c8c987256 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -149,7 +149,7 @@ static __init void setup_topology(void)
topology_max_mnest = max_mnest;
}
-static void early_pgm_check_handler(struct pt_regs *regs)
+void __do_early_pgm_check(struct pt_regs *regs)
{
if (!fixup_exception(regs))
disabled_wait();
@@ -159,12 +159,11 @@ static noinline __init void setup_lowcore_early(void)
{
psw_t psw;
- psw.addr = (unsigned long)s390_base_pgm_handler;
+ psw.addr = (unsigned long)early_pgm_check_handler;
psw.mask = PSW_MASK_BASE | PSW_DEFAULT_KEY | PSW_MASK_EA | PSW_MASK_BA;
if (IS_ENABLED(CONFIG_KASAN))
psw.mask |= PSW_MASK_DAT;
S390_lowcore.program_new_psw = psw;
- s390_base_pgm_handler_fn = early_pgm_check_handler;
S390_lowcore.preempt_count = INIT_PREEMPT_COUNT;
}
diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/earlypgm.S
similarity index 55%
rename from arch/s390/kernel/base.S
rename to arch/s390/kernel/earlypgm.S
index 172c23c8ca00..fbc7d2d538df 100644
--- a/arch/s390/kernel/base.S
+++ b/arch/s390/kernel/earlypgm.S
@@ -1,23 +1,16 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
- * arch/s390/kernel/base.S
- *
* Copyright IBM Corp. 2006, 2007
* Author(s): Michael Holzheu <holzheu@de.ibm.com>
*/
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
-#include <asm/nospec-insn.h>
-#include <asm/ptrace.h>
-
- GEN_BR_THUNK %r9
- GEN_BR_THUNK %r14
__PT_R0 = __PT_GPRS
__PT_R8 = __PT_GPRS + 64
-ENTRY(s390_base_pgm_handler)
+ENTRY(early_pgm_check_handler)
stmg %r8,%r15,__LC_SAVE_AREA_SYNC
aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE)
la %r11,STACK_FRAME_OVERHEAD(%r15)
@@ -26,25 +19,8 @@ ENTRY(s390_base_pgm_handler)
mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
lgr %r2,%r11
- larl %r1,s390_base_pgm_handler_fn
- lg %r9,0(%r1)
- ltgr %r9,%r9
- jz 1f
- BASR_EX %r14,%r9
+ brasl %r14,__do_early_pgm_check
mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
lpswe __LC_RETURN_PSW
-1: larl %r13,disabled_wait_psw
- lpswe 0(%r13)
-ENDPROC(s390_base_pgm_handler)
-
- .align 8
-disabled_wait_psw:
- .quad 0x0002000180000000,0x0000000000000000 + s390_base_pgm_handler
-
- .section .bss
- .align 8
- .globl s390_base_pgm_handler_fn
-s390_base_pgm_handler_fn:
- .quad 0
- .previous
+ENDPROC(early_pgm_check_handler)
diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h
index 56e5e3712fbb..995ec7449feb 100644
--- a/arch/s390/kernel/entry.h
+++ b/arch/s390/kernel/entry.h
@@ -17,10 +17,12 @@ void ext_int_handler(void);
void io_int_handler(void);
void mcck_int_handler(void);
void restart_int_handler(void);
+void early_pgm_check_handler(void);
void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs);
void __do_pgm_check(struct pt_regs *regs);
void __do_syscall(struct pt_regs *regs, int per_trap);
+void __do_early_pgm_check(struct pt_regs *regs);
void do_protection_exception(struct pt_regs *regs);
void do_dat_exception(struct pt_regs *regs);
From 3384f135e90f96a7697a85c33081e46c0a0f415d Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Mon, 23 May 2022 14:42:40 +0200
Subject: [PATCH 03/17] s390: generate register offsets into pt_regs
automatically
Use asm offsets method to generate register offsets into pt_regs,
instead of open-coding at several places.
Reviewed-by: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/kernel/asm-offsets.c | 16 ++++++++++++++++
arch/s390/kernel/earlypgm.S | 3 ---
arch/s390/kernel/entry.S | 17 -----------------
3 files changed, 16 insertions(+), 20 deletions(-)
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 7c74f0e17e5a..1e37cad44561 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -32,6 +32,22 @@ int main(void)
/* pt_regs offsets */
OFFSET(__PT_PSW, pt_regs, psw);
OFFSET(__PT_GPRS, pt_regs, gprs);
+ OFFSET(__PT_R0, pt_regs, gprs[0]);
+ OFFSET(__PT_R1, pt_regs, gprs[1]);
+ OFFSET(__PT_R2, pt_regs, gprs[2]);
+ OFFSET(__PT_R3, pt_regs, gprs[3]);
+ OFFSET(__PT_R4, pt_regs, gprs[4]);
+ OFFSET(__PT_R5, pt_regs, gprs[5]);
+ OFFSET(__PT_R6, pt_regs, gprs[6]);
+ OFFSET(__PT_R7, pt_regs, gprs[7]);
+ OFFSET(__PT_R8, pt_regs, gprs[8]);
+ OFFSET(__PT_R9, pt_regs, gprs[9]);
+ OFFSET(__PT_R10, pt_regs, gprs[10]);
+ OFFSET(__PT_R11, pt_regs, gprs[11]);
+ OFFSET(__PT_R12, pt_regs, gprs[12]);
+ OFFSET(__PT_R13, pt_regs, gprs[13]);
+ OFFSET(__PT_R14, pt_regs, gprs[14]);
+ OFFSET(__PT_R15, pt_regs, gprs[15]);
OFFSET(__PT_ORIG_GPR2, pt_regs, orig_gpr2);
OFFSET(__PT_FLAGS, pt_regs, flags);
OFFSET(__PT_CR1, pt_regs, cr1);
diff --git a/arch/s390/kernel/earlypgm.S b/arch/s390/kernel/earlypgm.S
index fbc7d2d538df..f521c6da37b8 100644
--- a/arch/s390/kernel/earlypgm.S
+++ b/arch/s390/kernel/earlypgm.S
@@ -7,9 +7,6 @@
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
-__PT_R0 = __PT_GPRS
-__PT_R8 = __PT_GPRS + 64
-
ENTRY(early_pgm_check_handler)
stmg %r8,%r15,__LC_SAVE_AREA_SYNC
aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE)
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index df41132ccd06..82faf4d0e4d7 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -29,23 +29,6 @@
#include <asm/export.h>
#include <asm/nospec-insn.h>
-__PT_R0 = __PT_GPRS
-__PT_R1 = __PT_GPRS + 8
-__PT_R2 = __PT_GPRS + 16
-__PT_R3 = __PT_GPRS + 24
-__PT_R4 = __PT_GPRS + 32
-__PT_R5 = __PT_GPRS + 40
-__PT_R6 = __PT_GPRS + 48
-__PT_R7 = __PT_GPRS + 56
-__PT_R8 = __PT_GPRS + 64
-__PT_R9 = __PT_GPRS + 72
-__PT_R10 = __PT_GPRS + 80
-__PT_R11 = __PT_GPRS + 88
-__PT_R12 = __PT_GPRS + 96
-__PT_R13 = __PT_GPRS + 104
-__PT_R14 = __PT_GPRS + 112
-__PT_R15 = __PT_GPRS + 120
-
STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
STACK_SIZE = 1 << STACK_SHIFT
STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
From c9bfb460c3e4da2462e16b0f0b200990b36b1dd2 Mon Sep 17 00:00:00 2001
From: Nico Boehr <nrb@linux.ibm.com>
Date: Tue, 24 May 2022 15:43:20 +0200
Subject: [PATCH 04/17] s390/perf: obtain sie_block from the right address
Since commit 1179f170b6f0 ("s390: fix fpu restore in entry.S"), the
sie_block pointer is located at empty1[1], but in sie_block() it was
taken from empty1[0].
This leads to a random pointer being dereferenced, possibly causing
system crash.
This problem can be observed when running a simple guest with an endless
loop and recording the cpu-clock event:
sudo perf kvm --guestvmlinux=<guestkernel> --guest top -e cpu-clock
With this fix, the correct guest address is shown.
Fixes: 1179f170b6f0 ("s390: fix fpu restore in entry.S")
Cc: stable@vger.kernel.org
Acked-by: Christian Borntraeger <borntraeger@de.ibm.com>
Acked-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
Reviewed-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/kernel/perf_event.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c
index ea7729bebaa0..a7f8db73984b 100644
--- a/arch/s390/kernel/perf_event.c
+++ b/arch/s390/kernel/perf_event.c
@@ -30,7 +30,7 @@ static struct kvm_s390_sie_block *sie_block(struct pt_regs *regs)
if (!stack)
return NULL;
- return (struct kvm_s390_sie_block *) stack->empty1[0];
+ return (struct kvm_s390_sie_block *)stack->empty1[1];
}
static bool is_in_guest(struct pt_regs *regs)
From 4d1f2e4576af57f8d6b357198dcdda88d0431d5e Mon Sep 17 00:00:00 2001
From: Juerg Haefliger <juerg.haefliger@canonical.com>
Date: Wed, 25 May 2022 14:01:40 +0200
Subject: [PATCH 05/17] s390/Kconfig: fix indentation
The convention for indentation seems to be a single tab. Help text is
further indented by an additional two whitespaces. Fix the lines that
violate these rules.
Signed-off-by: Juerg Haefliger <juerg.haefliger@canonical.com>
Link: https://lore.kernel.org/r/20220525120140.39534-1-juerg.haefliger@canonical.com
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/Kconfig | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index e084c72104f8..543e859905df 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -736,11 +736,11 @@ config VFIO_AP
depends on S390_AP_IOMMU && VFIO_MDEV && KVM
depends on ZCRYPT
help
- This driver grants access to Adjunct Processor (AP) devices
- via the VFIO mediated device interface.
+ This driver grants access to Adjunct Processor (AP) devices
+ via the VFIO mediated device interface.
- To compile this driver as a module, choose M here: the module
- will be called vfio_ap.
+ To compile this driver as a module, choose M here: the module
+ will be called vfio_ap.
endmenu
From 13e282cc92d7e4b96fad8b71adf48b7eab22f6b0 Mon Sep 17 00:00:00 2001
From: Juerg Haefliger <juerg.haefliger@canonical.com>
Date: Wed, 25 May 2022 14:01:51 +0200
Subject: [PATCH 06/17] s390/Kconfig.debug: fix indentation
The convention for indentation seems to be a single tab. Help text is
further indented by an additional two whitespaces. Fix the lines that
violate these rules.
Signed-off-by: Juerg Haefliger <juerg.haefliger@canonical.com>
Link: https://lore.kernel.org/r/20220525120151.39594-1-juerg.haefliger@canonical.com
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/Kconfig.debug | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/s390/Kconfig.debug b/arch/s390/Kconfig.debug
index e94a2a7f6bf4..c4300ea4abf8 100644
--- a/arch/s390/Kconfig.debug
+++ b/arch/s390/Kconfig.debug
@@ -14,9 +14,9 @@ config DEBUG_ENTRY
If unsure, say N.
config CIO_INJECT
- bool "CIO Inject interfaces"
- depends on DEBUG_KERNEL && DEBUG_FS
- help
- This option provides a debugging facility to inject certain artificial events
- and instruction responses to the CIO layer of Linux kernel. The newly created
- debugfs user-interfaces will be at /sys/kernel/debug/s390/cio/*
+ bool "CIO Inject interfaces"
+ depends on DEBUG_KERNEL && DEBUG_FS
+ help
+ This option provides a debugging facility to inject certain artificial events
+ and instruction responses to the CIO layer of Linux kernel. The newly created
+ debugfs user-interfaces will be at /sys/kernel/debug/s390/cio/*
From 9dade1707fdb4fb6a1d498aa8811779556b7ffbc Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Mon, 30 May 2022 11:37:48 +0200
Subject: [PATCH 07/17] s390/kexec: add __GFP_NORETRY to
KEXEC_CONTROL_MEMORY_GFP
Avoid invoking the OOM-killer when allocating the control page. This
is the s390 variant of commit dc5cccacf427 ("kexec: don't invoke
OOM-killer for control page allocation").
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/kexec.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h
index 7f3c9ac34bd8..9df3b4d64724 100644
--- a/arch/s390/include/asm/kexec.h
+++ b/arch/s390/include/asm/kexec.h
@@ -29,7 +29,7 @@
#define KEXEC_CONTROL_MEMORY_LIMIT (1UL<<31)
/* Allocate control page with GFP_DMA */
-#define KEXEC_CONTROL_MEMORY_GFP GFP_DMA
+#define KEXEC_CONTROL_MEMORY_GFP (GFP_DMA | __GFP_NORETRY)
/* Maximum address we can use for the crash control pages */
#define KEXEC_CRASH_CONTROL_MEMORY_LIMIT (-1UL)
From 315945a43e91c81ebfab0ef011d1dcab419c6d9b Mon Sep 17 00:00:00 2001
From: Eric Farman <farman@linux.ibm.com>
Date: Wed, 25 May 2022 16:40:28 +0200
Subject: [PATCH 08/17] MAINTAINERS: Update s390 virtio-ccw
Add myself to the kernel side of virtio-ccw
Signed-off-by: Eric Farman <farman@linux.ibm.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Acked-by: Halil Pasic <pasic@linux.ibm.com>
Link: https://lore.kernel.org/r/20220525144028.2714489-2-farman@linux.ibm.com
Signed-off-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
MAINTAINERS | 1 +
1 file changed, 1 insertion(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 5e8c2f611766..157bb82c65d1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -20862,6 +20862,7 @@ F: include/uapi/linux/virtio_crypto.h
VIRTIO DRIVERS FOR S390
M: Cornelia Huck <cohuck@redhat.com>
M: Halil Pasic <pasic@linux.ibm.com>
+M: Eric Farman <farman@linux.ibm.com>
L: linux-s390@vger.kernel.org
L: virtualization@lists.linux-foundation.org
L: kvm@vger.kernel.org
From 6d5946274df1fff539a7eece458a43be733d1db8 Mon Sep 17 00:00:00 2001
From: Christian Borntraeger <borntraeger@linux.ibm.com>
Date: Mon, 30 May 2022 11:27:05 +0200
Subject: [PATCH 09/17] s390/gmap: voluntarily schedule during key setting
With large and many guest with storage keys it is possible to create
large latencies or stalls during initial key setting:
rcu: INFO: rcu_sched self-detected stall on CPU
rcu: 18-....: (2099 ticks this GP) idle=54e/1/0x4000000000000002 softirq=35598716/35598716 fqs=998
(t=2100 jiffies g=155867385 q=20879)
Task dump for CPU 18:
CPU 1/KVM R running task 0 1030947 256019 0x06000004
Call Trace:
sched_show_task
rcu_dump_cpu_stacks
rcu_sched_clock_irq
update_process_times
tick_sched_handle
tick_sched_timer
__hrtimer_run_queues
hrtimer_interrupt
do_IRQ
ext_int_handler
ptep_zap_key
The mmap lock is held during the page walking but since this is a
semaphore scheduling is still possible. Same for the kvm srcu.
To minimize overhead do this on every segment table entry or large page.
Signed-off-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Reviewed-by: Alexander Gordeev <agordeev@linux.ibm.com>
Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
Link: https://lore.kernel.org/r/20220530092706.11637-2-borntraeger@linux.ibm.com
Signed-off-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/mm/gmap.c | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
index af03cacf34ec..f3694f6df8b7 100644
--- a/arch/s390/mm/gmap.c
+++ b/arch/s390/mm/gmap.c
@@ -2601,6 +2601,18 @@ static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr,
return 0;
}
+/*
+ * Give a chance to schedule after setting a key to 256 pages.
+ * We only hold the mm lock, which is a rwsem and the kvm srcu.
+ * Both can sleep.
+ */
+static int __s390_enable_skey_pmd(pmd_t *pmd, unsigned long addr,
+ unsigned long next, struct mm_walk *walk)
+{
+ cond_resched();
+ return 0;
+}
+
static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
unsigned long hmask, unsigned long next,
struct mm_walk *walk)
@@ -2623,12 +2635,14 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
end = start + HPAGE_SIZE - 1;
__storage_key_init_range(start, end);
set_bit(PG_arch_1, &page->flags);
+ cond_resched();
return 0;
}
static const struct mm_walk_ops enable_skey_walk_ops = {
.hugetlb_entry = __s390_enable_skey_hugetlb,
.pte_entry = __s390_enable_skey_pte,
+ .pmd_entry = __s390_enable_skey_pmd,
};
int s390_enable_skey(void)
From 3ae11dbcfac906a8c3a480e98660a823130dc16a Mon Sep 17 00:00:00 2001
From: Christian Borntraeger <borntraeger@linux.ibm.com>
Date: Mon, 30 May 2022 11:27:06 +0200
Subject: [PATCH 10/17] s390/mm: use non-quiescing sske for KVM switch to keyed
guest
The switch to a keyed guest does not require a classic sske as the other
guest CPUs are not accessing the key before the switch is complete.
By using the NQ SSKE things are faster especially with multiple guests.
Signed-off-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Suggested-by: Janis Schoetterl-Glausch <scgl@linux.ibm.com>
Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
Link: https://lore.kernel.org/r/20220530092706.11637-3-borntraeger@linux.ibm.com
Signed-off-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/mm/pgtable.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 697df02362af..4909dcd762e8 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -748,7 +748,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT;
ptev = pte_val(*ptep);
if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
- page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
+ page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0);
pgste_set_unlock(ptep, pgste);
preempt_enable();
}
From 29ccaa4b35ea874ddd50518e5c2c746b9238a792 Mon Sep 17 00:00:00 2001
From: Alexander Gordeev <agordeev@linux.ibm.com>
Date: Mon, 23 May 2022 12:38:14 +0200
Subject: [PATCH 11/17] s390/mcck: isolate SIE instruction when setting
CIF_MCCK_GUEST flag
Commit d768bd892fc8 ("s390: add options to change branch prediction
behaviour for the kernel") introduced .Lsie_exit label - supposedly
to fence off SIE instruction. However, the corresponding address
range length .Lsie_crit_mcck_length was not updated, which led to
BPON code potentionally marked with CIF_MCCK_GUEST flag.
Both .Lsie_exit and .Lsie_crit_mcck_length were removed with commit
0b0ed657fe00 ("s390: remove critical section cleanup from entry.S"),
but the issue persisted - currently BPOFF and BPENTER macros might
get wrongly considered by the machine check handler as a guest.
Fixes: d768bd892fc8 ("s390: add options to change branch prediction behaviour for the kernel")
Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/kernel/entry.S | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 82faf4d0e4d7..d2a1f2f4f5b8 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -251,6 +251,10 @@ ENTRY(sie64a)
BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
.Lsie_entry:
sie 0(%r14)
+# Let the next instruction be NOP to avoid triggering a machine check
+# and handling it in a guest as result of the instruction execution.
+ nopr 7
+.Lsie_leave:
BPOFF
BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
.Lsie_skip:
@@ -547,7 +551,7 @@ ENTRY(mcck_int_handler)
jno .Lmcck_panic
#if IS_ENABLED(CONFIG_KVM)
OUTSIDE %r9,.Lsie_gmap,.Lsie_done,6f
- OUTSIDE %r9,.Lsie_entry,.Lsie_skip,4f
+ OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f
oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
j 5f
4: CHKSTG .Lmcck_panic
From 79a74dac447be957178e8596706b1db3d5c714f4 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Sun, 29 May 2022 18:55:06 +0200
Subject: [PATCH 12/17] s390/uaccess: use symbolic names for inline assembler
operands
Make code easier to read by using symbolic names.
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/uaccess.h | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 1f150a7cfb3d..5ffb88b3fdee 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -278,12 +278,12 @@ int __noreturn __put_kernel_bad(void);
int __rc; \
\
asm volatile( \
- "0: " insn " %2,%1\n" \
- "1: xr %0,%0\n" \
+ "0: " insn " %[_val],%[_to]\n" \
+ "1: xr %[rc],%[rc]\n" \
"2:\n" \
EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \
- : "=d" (__rc), "+Q" (*(to)) \
- : "d" (val) \
+ : [rc] "=d" (__rc), [_to] "+Q" (*(to)) \
+ : [_val] "d" (val) \
: "cc"); \
__rc; \
})
@@ -321,12 +321,12 @@ int __noreturn __get_kernel_bad(void);
int __rc; \
\
asm volatile( \
- "0: " insn " %1,%2\n" \
- "1: xr %0,%0\n" \
+ "0: " insn " %[_val],%[_from]\n" \
+ "1: xr %[rc],%[rc]\n" \
"2:\n" \
EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \
- : "=d" (__rc), "+d" (val) \
- : "Q" (*(from)) \
+ : [rc] "=d" (__rc), [_val] "+d" (val) \
+ : [_from] "Q" (*(from)) \
: "cc"); \
__rc; \
})
From 454ede3f1424c36bc124f8d7ed4b3ea654654be1 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Sun, 1 May 2022 21:26:06 +0200
Subject: [PATCH 13/17] s390/uaccess: use exception handler to zero result on
get_user() failure
Historically the uaccess code pre-initializes the result of get_user()
(and now also __get_kernel_nofault()) to zero and uses the result as
input parameter for inline assemblies. This is different to what most,
if not all, other architectures are doing, which set the result to
zero within the exception handler in case of a fault.
Use the new extable mechanism and handle zeroing of the result within
the exception handler in case of a fault.
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/asm-extable.h | 91 ++++++++++++++++++++---------
arch/s390/include/asm/uaccess.h | 72 ++++++++++++++---------
arch/s390/mm/extable.c | 39 +++++++++++--
3 files changed, 143 insertions(+), 59 deletions(-)
diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
index f24d9591aaed..b74f1070ddb2 100644
--- a/arch/s390/include/asm/asm-extable.h
+++ b/arch/s390/include/asm/asm-extable.h
@@ -3,12 +3,24 @@
#define __ASM_EXTABLE_H
#include <linux/stringify.h>
+#include <linux/bits.h>
#include <asm/asm-const.h>
-#define EX_TYPE_NONE 0
-#define EX_TYPE_FIXUP 1
-#define EX_TYPE_BPF 2
-#define EX_TYPE_UACCESS 3
+#define EX_TYPE_NONE 0
+#define EX_TYPE_FIXUP 1
+#define EX_TYPE_BPF 2
+#define EX_TYPE_UA_STORE 3
+#define EX_TYPE_UA_LOAD_MEM 4
+#define EX_TYPE_UA_LOAD_REG 5
+
+#define EX_DATA_REG_ERR_SHIFT 0
+#define EX_DATA_REG_ERR GENMASK(3, 0)
+
+#define EX_DATA_REG_ADDR_SHIFT 4
+#define EX_DATA_REG_ADDR GENMASK(7, 4)
+
+#define EX_DATA_LEN_SHIFT 8
+#define EX_DATA_LEN GENMASK(11, 8)
#define __EX_TABLE(_section, _fault, _target, _type) \
stringify_in_c(.section _section,"a";) \
@@ -19,35 +31,58 @@
stringify_in_c(.short 0;) \
stringify_in_c(.previous)
-#define __EX_TABLE_UA(_section, _fault, _target, _type, _reg) \
- stringify_in_c(.section _section,"a";) \
- stringify_in_c(.align 4;) \
- stringify_in_c(.long (_fault) - .;) \
- stringify_in_c(.long (_target) - .;) \
- stringify_in_c(.short (_type);) \
- stringify_in_c(.macro extable_reg reg;) \
- stringify_in_c(.set .Lfound, 0;) \
- stringify_in_c(.set .Lregnr, 0;) \
- stringify_in_c(.irp rs,r0,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11,r12,r13,r14,r15;) \
- stringify_in_c(.ifc "\reg", "%%\rs";) \
- stringify_in_c(.set .Lfound, 1;) \
- stringify_in_c(.short .Lregnr;) \
- stringify_in_c(.endif;) \
- stringify_in_c(.set .Lregnr, .Lregnr+1;) \
- stringify_in_c(.endr;) \
- stringify_in_c(.ifne (.Lfound != 1);) \
- stringify_in_c(.error "extable_reg: bad register argument";) \
- stringify_in_c(.endif;) \
- stringify_in_c(.endm;) \
- stringify_in_c(extable_reg _reg;) \
- stringify_in_c(.purgem extable_reg;) \
+#define __EX_TABLE_UA(_section, _fault, _target, _type, _regerr, _regaddr, _len)\
+ stringify_in_c(.section _section,"a";) \
+ stringify_in_c(.align 4;) \
+ stringify_in_c(.long (_fault) - .;) \
+ stringify_in_c(.long (_target) - .;) \
+ stringify_in_c(.short (_type);) \
+ stringify_in_c(.macro extable_reg regerr, regaddr;) \
+ stringify_in_c(.set .Lfound, 0;) \
+ stringify_in_c(.set .Lcurr, 0;) \
+ stringify_in_c(.irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15;) \
+ stringify_in_c( .ifc "\regerr", "%%r\rs";) \
+ stringify_in_c( .set .Lfound, 1;) \
+ stringify_in_c( .set .Lregerr, .Lcurr;) \
+ stringify_in_c( .endif;) \
+ stringify_in_c( .set .Lcurr, .Lcurr+1;) \
+ stringify_in_c(.endr;) \
+ stringify_in_c(.ifne (.Lfound != 1);) \
+ stringify_in_c( .error "extable_reg: bad register argument1";) \
+ stringify_in_c(.endif;) \
+ stringify_in_c(.set .Lfound, 0;) \
+ stringify_in_c(.set .Lcurr, 0;) \
+ stringify_in_c(.irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15;) \
+ stringify_in_c( .ifc "\regaddr", "%%r\rs";) \
+ stringify_in_c( .set .Lfound, 1;) \
+ stringify_in_c( .set .Lregaddr, .Lcurr;) \
+ stringify_in_c( .endif;) \
+ stringify_in_c( .set .Lcurr, .Lcurr+1;) \
+ stringify_in_c(.endr;) \
+ stringify_in_c(.ifne (.Lfound != 1);) \
+ stringify_in_c( .error "extable_reg: bad register argument2";) \
+ stringify_in_c(.endif;) \
+ stringify_in_c(.short .Lregerr << EX_DATA_REG_ERR_SHIFT | \
+ .Lregaddr << EX_DATA_REG_ADDR_SHIFT | \
+ _len << EX_DATA_LEN_SHIFT;) \
+ stringify_in_c(.endm;) \
+ stringify_in_c(extable_reg _regerr,_regaddr;) \
+ stringify_in_c(.purgem extable_reg;) \
stringify_in_c(.previous)
#define EX_TABLE(_fault, _target) \
__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP)
+
#define EX_TABLE_AMODE31(_fault, _target) \
__EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP)
-#define EX_TABLE_UA(_fault, _target, _reg) \
- __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UACCESS, _reg)
+
+#define EX_TABLE_UA_STORE(_fault, _target, _regerr) \
+ __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0)
+
+#define EX_TABLE_UA_LOAD_MEM(_fault, _target, _regerr, _regmem, _len) \
+ __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len)
+
+#define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \
+ __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0)
#endif /* __ASM_EXTABLE_H */
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 5ffb88b3fdee..2d8d9e89d48f 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -80,8 +80,12 @@ union oac {
};
};
-#define __put_get_user_asm(to, from, size, oac_spec) \
+#define __put_user_asm(to, from, size) \
({ \
+ union oac __oac_spec = { \
+ .oac1.as = PSW_BITS_AS_SECONDARY, \
+ .oac1.a = 1, \
+ }; \
int __rc; \
\
asm volatile( \
@@ -89,26 +93,15 @@ union oac {
"0: mvcos %[_to],%[_from],%[_size]\n" \
"1: xr %[rc],%[rc]\n" \
"2:\n" \
- EX_TABLE_UA(0b,2b,%[rc]) EX_TABLE_UA(1b,2b,%[rc]) \
+ EX_TABLE_UA_STORE(0b, 2b, %[rc]) \
+ EX_TABLE_UA_STORE(1b, 2b, %[rc]) \
: [rc] "=&d" (__rc), [_to] "+Q" (*(to)) \
: [_size] "d" (size), [_from] "Q" (*(from)), \
- [spec] "d" (oac_spec.val) \
+ [spec] "d" (__oac_spec.val) \
: "cc", "0"); \
__rc; \
})
-#define __put_user_asm(to, from, size) \
- __put_get_user_asm(to, from, size, ((union oac) { \
- .oac1.as = PSW_BITS_AS_SECONDARY, \
- .oac1.a = 1 \
- }))
-
-#define __get_user_asm(to, from, size) \
- __put_get_user_asm(to, from, size, ((union oac) { \
- .oac2.as = PSW_BITS_AS_SECONDARY, \
- .oac2.a = 1 \
- })) \
-
static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size)
{
int rc;
@@ -141,6 +134,29 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon
return rc;
}
+#define __get_user_asm(to, from, size) \
+({ \
+ union oac __oac_spec = { \
+ .oac2.as = PSW_BITS_AS_SECONDARY, \
+ .oac2.a = 1, \
+ }; \
+ int __rc; \
+ \
+ asm volatile( \
+ " lr 0,%[spec]\n" \
+ "0: mvcos 0(%[_to]),%[_from],%[_size]\n" \
+ "1: xr %[rc],%[rc]\n" \
+ "2:\n" \
+ EX_TABLE_UA_LOAD_MEM(0b, 2b, %[rc], %[_to], %[_ksize]) \
+ EX_TABLE_UA_LOAD_MEM(1b, 2b, %[rc], %[_to], %[_ksize]) \
+ : [rc] "=&d" (__rc), "=Q" (*(to)) \
+ : [_size] "d" (size), [_from] "Q" (*(from)), \
+ [spec] "d" (__oac_spec.val), [_to] "a" (to), \
+ [_ksize] "K" (size) \
+ : "cc", "0"); \
+ __rc; \
+})
+
static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size)
{
int rc;
@@ -210,28 +226,28 @@ static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsign
__chk_user_ptr(ptr); \
switch (sizeof(*(ptr))) { \
case 1: { \
- unsigned char __x = 0; \
+ unsigned char __x; \
__gu_err = __get_user_fn(&__x, ptr, \
sizeof(*(ptr))); \
(x) = *(__force __typeof__(*(ptr)) *) &__x; \
break; \
}; \
case 2: { \
- unsigned short __x = 0; \
+ unsigned short __x; \
__gu_err = __get_user_fn(&__x, ptr, \
sizeof(*(ptr))); \
(x) = *(__force __typeof__(*(ptr)) *) &__x; \
break; \
}; \
case 4: { \
- unsigned int __x = 0; \
+ unsigned int __x; \
__gu_err = __get_user_fn(&__x, ptr, \
sizeof(*(ptr))); \
(x) = *(__force __typeof__(*(ptr)) *) &__x; \
break; \
}; \
case 8: { \
- unsigned long long __x = 0; \
+ unsigned long __x; \
__gu_err = __get_user_fn(&__x, ptr, \
sizeof(*(ptr))); \
(x) = *(__force __typeof__(*(ptr)) *) &__x; \
@@ -281,7 +297,8 @@ int __noreturn __put_kernel_bad(void);
"0: " insn " %[_val],%[_to]\n" \
"1: xr %[rc],%[rc]\n" \
"2:\n" \
- EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \
+ EX_TABLE_UA_STORE(0b, 2b, %[rc]) \
+ EX_TABLE_UA_STORE(1b, 2b, %[rc]) \
: [rc] "=d" (__rc), [_to] "+Q" (*(to)) \
: [_val] "d" (val) \
: "cc"); \
@@ -290,7 +307,7 @@ int __noreturn __put_kernel_bad(void);
#define __put_kernel_nofault(dst, src, type, err_label) \
do { \
- u64 __x = (u64)(*((type *)(src))); \
+ unsigned long __x = (unsigned long)(*((type *)(src))); \
int __pk_err; \
\
switch (sizeof(type)) { \
@@ -324,8 +341,9 @@ int __noreturn __get_kernel_bad(void);
"0: " insn " %[_val],%[_from]\n" \
"1: xr %[rc],%[rc]\n" \
"2:\n" \
- EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \
- : [rc] "=d" (__rc), [_val] "+d" (val) \
+ EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val]) \
+ EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val]) \
+ : [rc] "=d" (__rc), [_val] "=d" (val) \
: [_from] "Q" (*(from)) \
: "cc"); \
__rc; \
@@ -337,28 +355,28 @@ do { \
\
switch (sizeof(type)) { \
case 1: { \
- u8 __x = 0; \
+ unsigned char __x; \
\
__gk_err = __get_kernel_asm(__x, (type *)(src), "ic"); \
*((type *)(dst)) = (type)__x; \
break; \
}; \
case 2: { \
- u16 __x = 0; \
+ unsigned short __x; \
\
__gk_err = __get_kernel_asm(__x, (type *)(src), "lh"); \
*((type *)(dst)) = (type)__x; \
break; \
}; \
case 4: { \
- u32 __x = 0; \
+ unsigned int __x; \
\
__gk_err = __get_kernel_asm(__x, (type *)(src), "l"); \
*((type *)(dst)) = (type)__x; \
break; \
}; \
case 8: { \
- u64 __x = 0; \
+ unsigned long __x; \
\
__gk_err = __get_kernel_asm(__x, (type *)(src), "lg"); \
*((type *)(dst)) = (type)__x; \
diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
index 8ac8ad2474a0..1e4d2187541a 100644
--- a/arch/s390/mm/extable.c
+++ b/arch/s390/mm/extable.c
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
+#include <linux/bitfield.h>
#include <linux/extable.h>
+#include <linux/string.h>
#include <linux/errno.h>
#include <linux/panic.h>
#include <asm/asm-extable.h>
@@ -24,9 +26,34 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex, struct pt_r
return true;
}
-static bool ex_handler_uaccess(const struct exception_table_entry *ex, struct pt_regs *regs)
+static bool ex_handler_ua_store(const struct exception_table_entry *ex, struct pt_regs *regs)
{
- regs->gprs[ex->data] = -EFAULT;
+ unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+
+ regs->gprs[reg_err] = -EFAULT;
+ regs->psw.addr = extable_fixup(ex);
+ return true;
+}
+
+static bool ex_handler_ua_load_mem(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+ unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
+ unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+ size_t len = FIELD_GET(EX_DATA_LEN, ex->data);
+
+ regs->gprs[reg_err] = -EFAULT;
+ memset((void *)regs->gprs[reg_addr], 0, len);
+ regs->psw.addr = extable_fixup(ex);
+ return true;
+}
+
+static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+ unsigned int reg_zero = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
+ unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+
+ regs->gprs[reg_err] = -EFAULT;
+ regs->gprs[reg_zero] = 0;
regs->psw.addr = extable_fixup(ex);
return true;
}
@@ -43,8 +70,12 @@ bool fixup_exception(struct pt_regs *regs)
return ex_handler_fixup(ex, regs);
case EX_TYPE_BPF:
return ex_handler_bpf(ex, regs);
- case EX_TYPE_UACCESS:
- return ex_handler_uaccess(ex, regs);
+ case EX_TYPE_UA_STORE:
+ return ex_handler_ua_store(ex, regs);
+ case EX_TYPE_UA_LOAD_MEM:
+ return ex_handler_ua_load_mem(ex, regs);
+ case EX_TYPE_UA_LOAD_REG:
+ return ex_handler_ua_load_reg(ex, regs);
}
panic("invalid exception table entry");
}
From a0e3a44bc2a01153fe4ff56fc1ec26832a869437 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Sun, 29 May 2022 21:27:48 +0200
Subject: [PATCH 14/17] s390/uaccess: use __noreturn instead of
__attribute__((noreturn))
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/uaccess.h | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 2d8d9e89d48f..40361072e638 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -55,9 +55,6 @@ copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned lo
return n;
}
-int __put_user_bad(void) __attribute__((noreturn));
-int __get_user_bad(void) __attribute__((noreturn));
-
union oac {
unsigned int val;
struct {
@@ -80,6 +77,8 @@ union oac {
};
};
+int __noreturn __put_user_bad(void);
+
#define __put_user_asm(to, from, size) \
({ \
union oac __oac_spec = { \
@@ -134,6 +133,8 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon
return rc;
}
+int __noreturn __get_user_bad(void);
+
#define __get_user_asm(to, from, size) \
({ \
union oac __oac_spec = { \
From d144182ed19af0bed4e4a73b51c7af9caccf6fa2 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Sun, 29 May 2022 21:36:16 +0200
Subject: [PATCH 15/17] s390/uaccess: whitespace cleanup
Whitespace cleanup to get rid if some checkpatch findings, but mainly
to have consistent coding style within the header file again.
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/uaccess.h | 132 ++++++++++++++++----------------
1 file changed, 66 insertions(+), 66 deletions(-)
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 40361072e638..f4511e21d646 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -3,7 +3,7 @@
* S390 version
* Copyright IBM Corp. 1999, 2000
* Author(s): Hartmut Penner (hp@de.ibm.com),
- * Martin Schwidefsky (schwidefsky@de.ibm.com)
+ * Martin Schwidefsky (schwidefsky@de.ibm.com)
*
* Derived from "include/asm-i386/uaccess.h"
*/
@@ -194,77 +194,77 @@ static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsign
* These are the main single-value transfer routines. They automatically
* use the right size if we just have the right pointer type.
*/
-#define __put_user(x, ptr) \
-({ \
- __typeof__(*(ptr)) __x = (x); \
- int __pu_err = -EFAULT; \
- __chk_user_ptr(ptr); \
- switch (sizeof (*(ptr))) { \
- case 1: \
- case 2: \
- case 4: \
- case 8: \
- __pu_err = __put_user_fn(&__x, ptr, \
- sizeof(*(ptr))); \
- break; \
- default: \
- __put_user_bad(); \
- break; \
- } \
- __builtin_expect(__pu_err, 0); \
+#define __put_user(x, ptr) \
+({ \
+ __typeof__(*(ptr)) __x = (x); \
+ int __pu_err = -EFAULT; \
+ \
+ __chk_user_ptr(ptr); \
+ switch (sizeof(*(ptr))) { \
+ case 1: \
+ case 2: \
+ case 4: \
+ case 8: \
+ __pu_err = __put_user_fn(&__x, ptr, sizeof(*(ptr))); \
+ break; \
+ default: \
+ __put_user_bad(); \
+ break; \
+ } \
+ __builtin_expect(__pu_err, 0); \
})
-#define put_user(x, ptr) \
-({ \
- might_fault(); \
- __put_user(x, ptr); \
+#define put_user(x, ptr) \
+({ \
+ might_fault(); \
+ __put_user(x, ptr); \
})
-
-#define __get_user(x, ptr) \
-({ \
- int __gu_err = -EFAULT; \
- __chk_user_ptr(ptr); \
- switch (sizeof(*(ptr))) { \
- case 1: { \
- unsigned char __x; \
- __gu_err = __get_user_fn(&__x, ptr, \
- sizeof(*(ptr))); \
- (x) = *(__force __typeof__(*(ptr)) *) &__x; \
- break; \
- }; \
- case 2: { \
- unsigned short __x; \
- __gu_err = __get_user_fn(&__x, ptr, \
- sizeof(*(ptr))); \
- (x) = *(__force __typeof__(*(ptr)) *) &__x; \
- break; \
- }; \
- case 4: { \
- unsigned int __x; \
- __gu_err = __get_user_fn(&__x, ptr, \
- sizeof(*(ptr))); \
- (x) = *(__force __typeof__(*(ptr)) *) &__x; \
- break; \
- }; \
- case 8: { \
- unsigned long __x; \
- __gu_err = __get_user_fn(&__x, ptr, \
- sizeof(*(ptr))); \
- (x) = *(__force __typeof__(*(ptr)) *) &__x; \
- break; \
- }; \
- default: \
- __get_user_bad(); \
- break; \
- } \
- __builtin_expect(__gu_err, 0); \
+#define __get_user(x, ptr) \
+({ \
+ int __gu_err = -EFAULT; \
+ \
+ __chk_user_ptr(ptr); \
+ switch (sizeof(*(ptr))) { \
+ case 1: { \
+ unsigned char __x; \
+ \
+ __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \
+ (x) = *(__force __typeof__(*(ptr)) *)&__x; \
+ break; \
+ }; \
+ case 2: { \
+ unsigned short __x; \
+ \
+ __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \
+ (x) = *(__force __typeof__(*(ptr)) *)&__x; \
+ break; \
+ }; \
+ case 4: { \
+ unsigned int __x; \
+ \
+ __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \
+ (x) = *(__force __typeof__(*(ptr)) *)&__x; \
+ break; \
+ }; \
+ case 8: { \
+ unsigned long __x; \
+ \
+ __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \
+ (x) = *(__force __typeof__(*(ptr)) *)&__x; \
+ break; \
+ }; \
+ default: \
+ __get_user_bad(); \
+ break; \
+ } \
+ __builtin_expect(__gu_err, 0); \
})
-#define get_user(x, ptr) \
-({ \
- might_fault(); \
- __get_user(x, ptr); \
+#define get_user(x, ptr) \
+({ \
+ might_fault(); \
+ __get_user(x, ptr); \
})
/*
From f037acb41dc9fc0f00521685b3250226d6f9b437 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Mon, 30 May 2022 14:09:24 +0200
Subject: [PATCH 16/17] s390/stack: merge empty stack frame slots
Merge empty1 and empty2 arrays within the stack frame to one single
array. This is possible since with commit 42b01a553a56 ("s390: always
use the packed stack layout") the alternative stack frame layout is
gone.
Reviewed-by: Nico Boehr <nrb@linux.ibm.com>
Reviewed-by: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/stacktrace.h | 3 +--
arch/s390/kernel/asm-offsets.c | 10 +++++-----
arch/s390/kernel/perf_event.c | 2 +-
3 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h
index f8500191993d..7e0e15bee18d 100644
--- a/arch/s390/include/asm/stacktrace.h
+++ b/arch/s390/include/asm/stacktrace.h
@@ -39,8 +39,7 @@ static inline bool on_stack(struct stack_info *info,
* Kernel uses the packed stack layout (-mpacked-stack).
*/
struct stack_frame {
- unsigned long empty1[5];
- unsigned int empty2[8];
+ unsigned long empty[9];
unsigned long gprs[10];
unsigned long back_chain;
};
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 1e37cad44561..0e10d7ff4203 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -57,11 +57,11 @@ int main(void)
/* stack_frame offsets */
OFFSET(__SF_BACKCHAIN, stack_frame, back_chain);
OFFSET(__SF_GPRS, stack_frame, gprs);
- OFFSET(__SF_EMPTY, stack_frame, empty1[0]);
- OFFSET(__SF_SIE_CONTROL, stack_frame, empty1[1]);
- OFFSET(__SF_SIE_SAVEAREA, stack_frame, empty1[2]);
- OFFSET(__SF_SIE_REASON, stack_frame, empty1[3]);
- OFFSET(__SF_SIE_FLAGS, stack_frame, empty1[4]);
+ OFFSET(__SF_EMPTY, stack_frame, empty[0]);
+ OFFSET(__SF_SIE_CONTROL, stack_frame, empty[1]);
+ OFFSET(__SF_SIE_SAVEAREA, stack_frame, empty[2]);
+ OFFSET(__SF_SIE_REASON, stack_frame, empty[3]);
+ OFFSET(__SF_SIE_FLAGS, stack_frame, empty[4]);
DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame));
BLANK();
/* idle data offsets */
diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c
index a7f8db73984b..e259ff1f5ad3 100644
--- a/arch/s390/kernel/perf_event.c
+++ b/arch/s390/kernel/perf_event.c
@@ -30,7 +30,7 @@ static struct kvm_s390_sie_block *sie_block(struct pt_regs *regs)
if (!stack)
return NULL;
- return (struct kvm_s390_sie_block *)stack->empty1[1];
+ return (struct kvm_s390_sie_block *)stack->empty[1];
}
static bool is_in_guest(struct pt_regs *regs)
From e0ffcf3fe18e0310221461c08969edec2cc7628c Mon Sep 17 00:00:00 2001
From: Heiko Carstens <hca@linux.ibm.com>
Date: Mon, 30 May 2022 14:20:50 +0200
Subject: [PATCH 17/17] s390/stack: add union to reflect kvm stack slot usages
Add a union which describes how the empty stack slots are being used
by kvm and perf. This should help to avoid another bug like the one
which was fixed with commit c9bfb460c3e4 ("s390/perf: obtain sie_block
from the right address").
Reviewed-by: Nico Boehr <nrb@linux.ibm.com>
Tested-by: Nico Boehr <nrb@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/include/asm/stacktrace.h | 10 +++++++++-
arch/s390/kernel/asm-offsets.c | 8 ++++----
arch/s390/kernel/perf_event.c | 2 +-
3 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h
index 7e0e15bee18d..b23c658dce77 100644
--- a/arch/s390/include/asm/stacktrace.h
+++ b/arch/s390/include/asm/stacktrace.h
@@ -39,7 +39,15 @@ static inline bool on_stack(struct stack_info *info,
* Kernel uses the packed stack layout (-mpacked-stack).
*/
struct stack_frame {
- unsigned long empty[9];
+ union {
+ unsigned long empty[9];
+ struct {
+ unsigned long sie_control_block;
+ unsigned long sie_savearea;
+ unsigned long sie_reason;
+ unsigned long sie_flags;
+ };
+ };
unsigned long gprs[10];
unsigned long back_chain;
};
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 0e10d7ff4203..d8ce965c0a97 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -58,10 +58,10 @@ int main(void)
OFFSET(__SF_BACKCHAIN, stack_frame, back_chain);
OFFSET(__SF_GPRS, stack_frame, gprs);
OFFSET(__SF_EMPTY, stack_frame, empty[0]);
- OFFSET(__SF_SIE_CONTROL, stack_frame, empty[1]);
- OFFSET(__SF_SIE_SAVEAREA, stack_frame, empty[2]);
- OFFSET(__SF_SIE_REASON, stack_frame, empty[3]);
- OFFSET(__SF_SIE_FLAGS, stack_frame, empty[4]);
+ OFFSET(__SF_SIE_CONTROL, stack_frame, sie_control_block);
+ OFFSET(__SF_SIE_SAVEAREA, stack_frame, sie_savearea);
+ OFFSET(__SF_SIE_REASON, stack_frame, sie_reason);
+ OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags);
DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame));
BLANK();
/* idle data offsets */
diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c
index e259ff1f5ad3..c27321cb0969 100644
--- a/arch/s390/kernel/perf_event.c
+++ b/arch/s390/kernel/perf_event.c
@@ -30,7 +30,7 @@ static struct kvm_s390_sie_block *sie_block(struct pt_regs *regs)
if (!stack)
return NULL;
- return (struct kvm_s390_sie_block *)stack->empty[1];
+ return (struct kvm_s390_sie_block *)stack->sie_control_block;
}
static bool is_in_guest(struct pt_regs *regs)