2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* S390 low-level entry points.
|
|
|
|
*
|
2012-07-20 09:15:04 +00:00
|
|
|
* Copyright IBM Corp. 1999, 2012
|
2005-04-16 22:20:36 +00:00
|
|
|
* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
|
2006-09-28 14:56:37 +00:00
|
|
|
* Hartmut Penner (hp@de.ibm.com),
|
|
|
|
* Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
|
2005-06-25 21:55:30 +00:00
|
|
|
* Heiko Carstens <heiko.carstens@de.ibm.com>
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2008-02-05 15:50:40 +00:00
|
|
|
#include <linux/init.h>
|
2011-07-24 08:48:19 +00:00
|
|
|
#include <linux/linkage.h>
|
2012-09-05 11:26:11 +00:00
|
|
|
#include <asm/processor.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/cache.h>
|
|
|
|
#include <asm/errno.h>
|
|
|
|
#include <asm/ptrace.h>
|
|
|
|
#include <asm/thread_info.h>
|
2005-09-09 18:57:26 +00:00
|
|
|
#include <asm/asm-offsets.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/unistd.h>
|
|
|
|
#include <asm/page.h>
|
2012-06-04 13:05:43 +00:00
|
|
|
#include <asm/sigp.h>
|
2013-06-27 07:01:09 +00:00
|
|
|
#include <asm/irq.h>
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
#include <asm/fpu-internal.h>
|
|
|
|
#include <asm/vx-insn.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-12-27 10:27:15 +00:00
|
|
|
__PT_R0 = __PT_GPRS
|
|
|
|
__PT_R1 = __PT_GPRS + 8
|
|
|
|
__PT_R2 = __PT_GPRS + 16
|
|
|
|
__PT_R3 = __PT_GPRS + 24
|
|
|
|
__PT_R4 = __PT_GPRS + 32
|
|
|
|
__PT_R5 = __PT_GPRS + 40
|
|
|
|
__PT_R6 = __PT_GPRS + 48
|
|
|
|
__PT_R7 = __PT_GPRS + 56
|
|
|
|
__PT_R8 = __PT_GPRS + 64
|
|
|
|
__PT_R9 = __PT_GPRS + 72
|
|
|
|
__PT_R10 = __PT_GPRS + 80
|
|
|
|
__PT_R11 = __PT_GPRS + 88
|
|
|
|
__PT_R12 = __PT_GPRS + 96
|
|
|
|
__PT_R13 = __PT_GPRS + 104
|
|
|
|
__PT_R14 = __PT_GPRS + 112
|
|
|
|
__PT_R15 = __PT_GPRS + 120
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
|
|
|
|
STACK_SIZE = 1 << STACK_SHIFT
|
2013-04-24 08:20:43 +00:00
|
|
|
STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-09-22 14:39:06 +00:00
|
|
|
_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
|
|
|
|
_TIF_UPROBE)
|
2014-04-15 10:55:07 +00:00
|
|
|
_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
|
|
|
|
_TIF_SYSCALL_TRACEPOINT)
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE | _CIF_FPU)
|
2014-04-15 10:55:07 +00:00
|
|
|
_PIF_WORK = (_PIF_PER_TRAP)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
#define BASED(name) name-cleanup_critical(%r13)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-07-03 07:24:46 +00:00
|
|
|
.macro TRACE_IRQS_ON
|
2011-12-27 10:27:15 +00:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
2010-05-17 08:00:02 +00:00
|
|
|
basr %r2,%r0
|
|
|
|
brasl %r14,trace_hardirqs_on_caller
|
2011-12-27 10:27:15 +00:00
|
|
|
#endif
|
2006-07-03 07:24:46 +00:00
|
|
|
.endm
|
|
|
|
|
|
|
|
.macro TRACE_IRQS_OFF
|
2011-12-27 10:27:15 +00:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
2010-05-17 08:00:02 +00:00
|
|
|
basr %r2,%r0
|
|
|
|
brasl %r14,trace_hardirqs_off_caller
|
2007-11-20 10:13:32 +00:00
|
|
|
#endif
|
2011-12-27 10:27:15 +00:00
|
|
|
.endm
|
2007-11-20 10:13:32 +00:00
|
|
|
|
|
|
|
.macro LOCKDEP_SYS_EXIT
|
2011-12-27 10:27:15 +00:00
|
|
|
#ifdef CONFIG_LOCKDEP
|
|
|
|
tm __PT_PSW+1(%r11),0x01 # returning to user ?
|
|
|
|
jz .+10
|
2007-11-20 10:13:32 +00:00
|
|
|
brasl %r14,lockdep_sys_exit
|
2006-07-03 07:24:46 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
|
|
|
|
2011-12-27 10:27:15 +00:00
|
|
|
.macro CHECK_STACK stacksize,savearea
|
2006-06-29 12:58:05 +00:00
|
|
|
#ifdef CONFIG_CHECK_STACK
|
2011-12-27 10:27:15 +00:00
|
|
|
tml %r15,\stacksize - CONFIG_STACK_GUARD
|
|
|
|
lghi %r14,\savearea
|
|
|
|
jz stack_overflow
|
2006-06-29 12:58:05 +00:00
|
|
|
#endif
|
|
|
|
.endm
|
|
|
|
|
2015-06-22 15:28:14 +00:00
|
|
|
.macro SWITCH_ASYNC savearea,timer
|
2011-12-27 10:27:15 +00:00
|
|
|
tmhh %r8,0x0001 # interrupting from user ?
|
|
|
|
jnz 1f
|
|
|
|
lgr %r14,%r9
|
|
|
|
slg %r14,BASED(.Lcritical_start)
|
|
|
|
clg %r14,BASED(.Lcritical_length)
|
2005-04-16 22:20:36 +00:00
|
|
|
jhe 0f
|
2011-12-27 10:27:15 +00:00
|
|
|
lghi %r11,\savearea # inside critical section, do cleanup
|
2005-04-16 22:20:36 +00:00
|
|
|
brasl %r14,cleanup_critical
|
2011-12-27 10:27:15 +00:00
|
|
|
tmhh %r8,0x0001 # retest problem state after cleanup
|
2005-04-16 22:20:36 +00:00
|
|
|
jnz 1f
|
2015-06-22 15:28:14 +00:00
|
|
|
0: lg %r14,__LC_ASYNC_STACK # are we already on the async stack?
|
2005-04-16 22:20:36 +00:00
|
|
|
slgr %r14,%r15
|
2015-06-22 15:28:14 +00:00
|
|
|
srag %r14,%r14,STACK_SHIFT
|
2015-06-22 15:27:48 +00:00
|
|
|
jnz 2f
|
2015-06-22 15:28:14 +00:00
|
|
|
CHECK_STACK 1<<STACK_SHIFT,\savearea
|
2013-04-24 08:20:43 +00:00
|
|
|
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
2015-06-22 15:27:48 +00:00
|
|
|
j 3f
|
|
|
|
1: LAST_BREAK %r14
|
|
|
|
UPDATE_VTIME %r14,%r15,\timer
|
2015-06-22 15:28:14 +00:00
|
|
|
2: lg %r15,__LC_ASYNC_STACK # load async stack
|
2015-06-22 15:27:48 +00:00
|
|
|
3: la %r11,STACK_FRAME_OVERHEAD(%r15)
|
2006-09-28 14:56:37 +00:00
|
|
|
.endm
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-06-22 15:27:48 +00:00
|
|
|
.macro UPDATE_VTIME w1,w2,enter_timer
|
|
|
|
lg \w1,__LC_EXIT_TIMER
|
|
|
|
lg \w2,__LC_LAST_UPDATE_TIMER
|
|
|
|
slg \w1,\enter_timer
|
|
|
|
slg \w2,__LC_EXIT_TIMER
|
|
|
|
alg \w1,__LC_USER_TIMER
|
|
|
|
alg \w2,__LC_SYSTEM_TIMER
|
|
|
|
stg \w1,__LC_USER_TIMER
|
|
|
|
stg \w2,__LC_SYSTEM_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
|
|
|
|
2011-12-27 10:27:15 +00:00
|
|
|
.macro LAST_BREAK scratch
|
|
|
|
srag \scratch,%r10,23
|
|
|
|
jz .+10
|
|
|
|
stg %r10,__TI_last_break(%r12)
|
2010-05-17 08:00:05 +00:00
|
|
|
.endm
|
|
|
|
|
2010-10-25 14:10:37 +00:00
|
|
|
.macro REENABLE_IRQS
|
2011-12-27 10:27:15 +00:00
|
|
|
stg %r8,__LC_RETURN_PSW
|
|
|
|
ni __LC_RETURN_PSW,0xbf
|
|
|
|
ssm __LC_RETURN_PSW
|
2010-10-25 14:10:37 +00:00
|
|
|
.endm
|
|
|
|
|
2012-05-09 14:27:39 +00:00
|
|
|
.macro STCK savearea
|
2012-05-14 08:35:22 +00:00
|
|
|
#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
|
2012-05-09 14:27:39 +00:00
|
|
|
.insn s,0xb27c0000,\savearea # store clock fast
|
|
|
|
#else
|
|
|
|
.insn s,0xb2050000,\savearea # store clock
|
|
|
|
#endif
|
|
|
|
.endm
|
|
|
|
|
2011-01-05 11:47:25 +00:00
|
|
|
.section .kprobes.text, "ax"
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Scheduler resume function, called by switch_to
|
|
|
|
* gpr2 = (task_struct *) prev
|
|
|
|
* gpr3 = (task_struct *) next
|
|
|
|
* Returns:
|
|
|
|
* gpr2 = prev
|
|
|
|
*/
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(__switch_to)
|
2012-05-15 07:20:06 +00:00
|
|
|
stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
|
2015-07-20 08:01:46 +00:00
|
|
|
lgr %r1,%r2
|
|
|
|
aghi %r1,__TASK_thread # thread_struct of prev task
|
|
|
|
lg %r4,__TASK_thread_info(%r2) # get thread_info of prev
|
|
|
|
lg %r5,__TASK_thread_info(%r3) # get thread_info of next
|
|
|
|
stg %r15,__THREAD_ksp(%r1) # store kernel stack of prev
|
|
|
|
lgr %r1,%r3
|
|
|
|
aghi %r1,__TASK_thread # thread_struct of next task
|
2012-05-15 07:20:06 +00:00
|
|
|
lgr %r15,%r5
|
2013-04-24 08:20:43 +00:00
|
|
|
aghi %r15,STACK_INIT # end of kernel stack of next
|
2012-05-15 07:20:06 +00:00
|
|
|
stg %r3,__LC_CURRENT # store task struct of next
|
|
|
|
stg %r5,__LC_THREAD_INFO # store thread info of next
|
|
|
|
stg %r15,__LC_KERNEL_STACK # store end of kernel stack
|
2015-07-20 08:01:46 +00:00
|
|
|
lg %r15,__THREAD_ksp(%r1) # load kernel stack of next
|
2012-05-15 07:20:06 +00:00
|
|
|
lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
|
|
|
|
mvc __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next
|
2014-04-15 10:55:07 +00:00
|
|
|
lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
|
2005-04-16 22:20:36 +00:00
|
|
|
br %r14
|
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.L__critical_start:
|
2015-06-22 15:26:40 +00:00
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_KVM)
|
|
|
|
/*
|
|
|
|
* sie64a calling convention:
|
|
|
|
* %r2 pointer to sie control block
|
|
|
|
* %r3 guest register save area
|
|
|
|
*/
|
|
|
|
ENTRY(sie64a)
|
|
|
|
stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
|
|
|
|
stg %r2,__SF_EMPTY(%r15) # save control block pointer
|
|
|
|
stg %r3,__SF_EMPTY+8(%r15) # save guest register save area
|
|
|
|
xc __SF_EMPTY+16(16,%r15),__SF_EMPTY+16(%r15) # host id & reason
|
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU # load guest fp/vx registers ?
|
|
|
|
jno .Lsie_load_guest_gprs
|
|
|
|
brasl %r14,load_fpu_regs # load guest fp/vx regs
|
|
|
|
.Lsie_load_guest_gprs:
|
|
|
|
lmg %r0,%r13,0(%r3) # load guest gprs 0-13
|
|
|
|
lg %r14,__LC_GMAP # get gmap pointer
|
|
|
|
ltgr %r14,%r14
|
|
|
|
jz .Lsie_gmap
|
|
|
|
lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
|
|
|
|
.Lsie_gmap:
|
|
|
|
lg %r14,__SF_EMPTY(%r15) # get control block pointer
|
|
|
|
oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
|
|
|
|
tm __SIE_PROG20+3(%r14),3 # last exit...
|
|
|
|
jnz .Lsie_skip
|
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
jo .Lsie_skip # exit if fp/vx regs changed
|
|
|
|
tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_LPP
|
|
|
|
jz .Lsie_enter
|
2015-07-09 15:56:13 +00:00
|
|
|
.insn s,0xb2800000,__LC_CURRENT_PID # set guest id to pid
|
2015-06-22 15:26:40 +00:00
|
|
|
.Lsie_enter:
|
|
|
|
sie 0(%r14)
|
|
|
|
tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_LPP
|
|
|
|
jz .Lsie_skip
|
|
|
|
.insn s,0xb2800000,__SF_EMPTY+16(%r15)# set host id
|
|
|
|
.Lsie_skip:
|
|
|
|
ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
|
|
|
|
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
|
|
|
.Lsie_done:
|
|
|
|
# some program checks are suppressing. C code (e.g. do_protection_exception)
|
|
|
|
# will rewind the PSW by the ILC, which is 4 bytes in case of SIE. Other
|
|
|
|
# instructions between sie64a and .Lsie_done should not cause program
|
|
|
|
# interrupts. So lets use a nop (47 00 00 00) as a landing pad.
|
|
|
|
# See also .Lcleanup_sie
|
|
|
|
.Lrewind_pad:
|
|
|
|
nop 0
|
|
|
|
.globl sie_exit
|
|
|
|
sie_exit:
|
|
|
|
lg %r14,__SF_EMPTY+8(%r15) # load guest register save area
|
|
|
|
stmg %r0,%r13,0(%r14) # save guest gprs 0-13
|
|
|
|
lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
|
|
|
|
lg %r2,__SF_EMPTY+24(%r15) # return exit reason code
|
|
|
|
br %r14
|
|
|
|
.Lsie_fault:
|
|
|
|
lghi %r14,-EFAULT
|
|
|
|
stg %r14,__SF_EMPTY+24(%r15) # set exit reason code
|
|
|
|
j sie_exit
|
|
|
|
|
|
|
|
EX_TABLE(.Lrewind_pad,.Lsie_fault)
|
|
|
|
EX_TABLE(sie_exit,.Lsie_fault)
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* SVC interrupt handler routine. System calls are synchronous events and
|
|
|
|
* are executed with interrupts enabled.
|
|
|
|
*/
|
|
|
|
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(system_call)
|
2008-12-25 12:39:25 +00:00
|
|
|
stpt __LC_SYNC_ENTER_TIMER
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_stmg:
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r8,%r15,__LC_SAVE_AREA_SYNC
|
|
|
|
lg %r10,__LC_LAST_BREAK
|
|
|
|
lg %r12,__LC_THREAD_INFO
|
2014-04-15 10:55:07 +00:00
|
|
|
lghi %r14,_PIF_SYSCALL
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_per:
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r15,__LC_KERNEL_STACK
|
|
|
|
la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
|
|
|
|
LAST_BREAK %r13
|
2015-06-22 15:27:48 +00:00
|
|
|
.Lsysc_vtime:
|
|
|
|
UPDATE_VTIME %r10,%r13,__LC_SYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r0,%r7,__PT_R0(%r11)
|
|
|
|
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
|
|
|
|
mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
|
2011-12-27 10:27:18 +00:00
|
|
|
mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
|
2014-04-15 10:55:07 +00:00
|
|
|
stg %r14,__PT_FLAGS(%r11)
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_do_svc:
|
2013-04-24 10:58:39 +00:00
|
|
|
lg %r10,__TI_sysc_table(%r12) # address of system call table
|
2011-12-27 10:27:18 +00:00
|
|
|
llgh %r8,__PT_INT_CODE+2(%r11)
|
2011-12-27 10:27:15 +00:00
|
|
|
slag %r8,%r8,2 # shift and test for svc 0
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lsysc_nr_ok
|
2005-04-16 22:20:36 +00:00
|
|
|
# svc 0: system call number in %r1
|
2011-12-27 10:27:15 +00:00
|
|
|
llgfr %r1,%r1 # clear high word in r1
|
2010-05-17 08:00:05 +00:00
|
|
|
cghi %r1,NR_syscalls
|
2014-12-03 16:00:08 +00:00
|
|
|
jnl .Lsysc_nr_ok
|
2011-12-27 10:27:18 +00:00
|
|
|
sth %r1,__PT_INT_CODE+2(%r11)
|
2011-12-27 10:27:15 +00:00
|
|
|
slag %r8,%r1,2
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_nr_ok:
|
2011-12-27 10:27:15 +00:00
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
|
|
|
stg %r2,__PT_ORIG_GPR2(%r11)
|
|
|
|
stg %r7,STACK_FRAME_OVERHEAD(%r15)
|
|
|
|
lgf %r9,0(%r8,%r10) # get system call add.
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_TRACE
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lsysc_tracesys
|
2011-12-27 10:27:15 +00:00
|
|
|
basr %r14,%r9 # call sys_xxxx
|
|
|
|
stg %r2,__PT_R2(%r11) # store return value
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_return:
|
2010-05-17 08:00:02 +00:00
|
|
|
LOCKDEP_SYS_EXIT
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_tif:
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __PT_FLAGS+7(%r11),_PIF_WORK
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lsysc_work
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_WORK
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lsysc_work # check for work
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_WORK
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lsysc_work
|
|
|
|
.Lsysc_restore:
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r14,__LC_VDSO_PER_CPU
|
|
|
|
lmg %r0,%r10,__PT_R0(%r11)
|
|
|
|
mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
|
|
|
|
stpt __LC_EXIT_TIMER
|
|
|
|
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
|
|
|
lmg %r11,%r15,__PT_R11(%r11)
|
|
|
|
lpswe __LC_RETURN_PSW
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_done:
|
2007-11-20 10:13:32 +00:00
|
|
|
|
2010-05-17 08:00:01 +00:00
|
|
|
#
|
|
|
|
# One of the work bits is on. Find out which one.
|
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_work:
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_mcck_pending
|
2010-05-17 08:00:05 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_NEED_RESCHED
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_reschedule
|
2014-09-22 14:39:06 +00:00
|
|
|
#ifdef CONFIG_UPROBES
|
|
|
|
tm __TI_flags+7(%r12),_TIF_UPROBE
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_uprobe_notify
|
2014-09-22 14:39:06 +00:00
|
|
|
#endif
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __PT_FLAGS+7(%r11),_PIF_PER_TRAP
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_singlestep
|
2010-05-17 08:00:05 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_SIGPENDING
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_sigpending
|
2010-05-17 08:00:05 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_notify_resume
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
jo .Lsysc_vxrs
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_ASCE
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lsysc_uaccess
|
|
|
|
j .Lsysc_return # beware of critical section cleanup
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#
|
|
|
|
# _TIF_NEED_RESCHED is set, call schedule
|
2006-09-28 14:56:37 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_reschedule:
|
|
|
|
larl %r14,.Lsysc_return
|
2011-12-27 10:27:15 +00:00
|
|
|
jg schedule
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-06-25 21:55:30 +00:00
|
|
|
#
|
2014-04-15 10:55:07 +00:00
|
|
|
# _CIF_MCCK_PENDING is set, call handler
|
2005-06-25 21:55:30 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_mcck_pending:
|
|
|
|
larl %r14,.Lsysc_return
|
2006-09-28 14:56:37 +00:00
|
|
|
jg s390_handle_mcck # TIF bit will be cleared by handler
|
2005-06-25 21:55:30 +00:00
|
|
|
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
#
|
2014-04-15 10:55:07 +00:00
|
|
|
# _CIF_ASCE is set, load user space asce
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_uaccess:
|
2014-04-15 10:55:07 +00:00
|
|
|
ni __LC_CPU_FLAGS+7,255-_CIF_ASCE
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lsysc_return
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
#
|
|
|
|
# CIF_FPU is set, restore floating-point controls and floating-point registers.
|
|
|
|
#
|
|
|
|
.Lsysc_vxrs:
|
|
|
|
larl %r14,.Lsysc_return
|
|
|
|
jg load_fpu_regs
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2008-04-30 07:53:08 +00:00
|
|
|
# _TIF_SIGPENDING is set, call do_signal
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_sigpending:
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
|
|
|
brasl %r14,do_signal
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __PT_FLAGS+7(%r11),_PIF_SYSCALL
|
2014-12-03 16:00:08 +00:00
|
|
|
jno .Lsysc_return
|
2011-12-27 10:27:15 +00:00
|
|
|
lmg %r2,%r7,__PT_R2(%r11) # load svc arguments
|
2013-09-27 13:24:38 +00:00
|
|
|
lg %r10,__TI_sysc_table(%r12) # address of system call table
|
2011-12-27 10:27:15 +00:00
|
|
|
lghi %r8,0 # svc 0 returns -ENOSYS
|
2012-10-09 11:33:53 +00:00
|
|
|
llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number
|
2011-10-30 14:16:49 +00:00
|
|
|
cghi %r1,NR_syscalls
|
2014-12-03 16:00:08 +00:00
|
|
|
jnl .Lsysc_nr_ok # invalid svc number -> do svc 0
|
2011-12-27 10:27:15 +00:00
|
|
|
slag %r8,%r1,2
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lsysc_nr_ok # restart svc
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-10-10 19:33:20 +00:00
|
|
|
#
|
|
|
|
# _TIF_NOTIFY_RESUME is set, call do_notify_resume
|
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_notify_resume:
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r14,.Lsysc_return
|
2011-12-27 10:27:15 +00:00
|
|
|
jg do_notify_resume
|
2008-10-10 19:33:20 +00:00
|
|
|
|
2014-09-22 14:39:06 +00:00
|
|
|
#
|
|
|
|
# _TIF_UPROBE is set, call uprobe_notify_resume
|
|
|
|
#
|
|
|
|
#ifdef CONFIG_UPROBES
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_uprobe_notify:
|
2014-09-22 14:39:06 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r14,.Lsysc_return
|
2014-09-22 14:39:06 +00:00
|
|
|
jg uprobe_notify_resume
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2014-04-15 10:55:07 +00:00
|
|
|
# _PIF_PER_TRAP is set, call do_per_trap
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_singlestep:
|
2014-04-15 10:55:07 +00:00
|
|
|
ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r14,.Lsysc_return
|
2011-01-05 11:48:10 +00:00
|
|
|
jg do_per_trap
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#
|
2008-10-10 19:33:20 +00:00
|
|
|
# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
|
|
|
|
# and after the system call
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_tracesys:
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2005-04-16 22:20:36 +00:00
|
|
|
la %r3,0
|
2011-12-27 10:27:18 +00:00
|
|
|
llgh %r0,__PT_INT_CODE+2(%r11)
|
2011-12-27 10:27:15 +00:00
|
|
|
stg %r0,__PT_R2(%r11)
|
2008-10-10 19:33:20 +00:00
|
|
|
brasl %r14,do_syscall_trace_enter
|
2005-04-16 22:20:36 +00:00
|
|
|
lghi %r0,NR_syscalls
|
2008-10-10 19:33:20 +00:00
|
|
|
clgr %r0,%r2
|
2014-12-03 16:00:08 +00:00
|
|
|
jnh .Lsysc_tracenogo
|
2011-12-27 10:27:15 +00:00
|
|
|
sllg %r8,%r2,2
|
|
|
|
lgf %r9,0(%r8,%r10)
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_tracego:
|
2011-12-27 10:27:15 +00:00
|
|
|
lmg %r3,%r7,__PT_R3(%r11)
|
|
|
|
stg %r7,STACK_FRAME_OVERHEAD(%r15)
|
|
|
|
lg %r2,__PT_ORIG_GPR2(%r11)
|
|
|
|
basr %r14,%r9 # call sys_xxx
|
|
|
|
stg %r2,__PT_R2(%r11) # store return value
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lsysc_tracenogo:
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_TRACE
|
2014-12-03 16:00:08 +00:00
|
|
|
jz .Lsysc_return
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r14,.Lsysc_return
|
2008-10-10 19:33:20 +00:00
|
|
|
jg do_syscall_trace_exit
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#
|
|
|
|
# a new process exits the kernel with ret_from_fork
|
|
|
|
#
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(ret_from_fork)
|
2011-12-27 10:27:15 +00:00
|
|
|
la %r11,STACK_FRAME_OVERHEAD(%r15)
|
|
|
|
lg %r12,__LC_THREAD_INFO
|
2012-09-10 22:03:41 +00:00
|
|
|
brasl %r14,schedule_tail
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
ssm __LC_SVC_NEW_PSW # reenable interrupts
|
2012-10-11 19:30:14 +00:00
|
|
|
tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
|
2014-12-03 16:00:08 +00:00
|
|
|
jne .Lsysc_tracenogo
|
2012-10-11 19:30:14 +00:00
|
|
|
# it's a kernel thread
|
|
|
|
lmg %r9,%r10,__PT_R9(%r11) # load gprs
|
2012-09-10 22:03:41 +00:00
|
|
|
ENTRY(kernel_thread_starter)
|
|
|
|
la %r2,0(%r10)
|
|
|
|
basr %r14,%r9
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lsysc_tracenogo
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Program check handler routine
|
|
|
|
*/
|
|
|
|
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(pgm_check_handler)
|
2008-12-25 12:39:25 +00:00
|
|
|
stpt __LC_SYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r8,%r15,__LC_SAVE_AREA_SYNC
|
|
|
|
lg %r10,__LC_LAST_BREAK
|
|
|
|
lg %r12,__LC_THREAD_INFO
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
larl %r13,cleanup_critical
|
2011-12-27 10:27:15 +00:00
|
|
|
lmg %r8,%r9,__LC_PGM_OLD_PSW
|
|
|
|
tmhh %r8,0x0001 # test problem state bit
|
2015-06-22 15:26:40 +00:00
|
|
|
jnz 2f # -> fault in user space
|
|
|
|
#if IS_ENABLED(CONFIG_KVM)
|
|
|
|
# cleanup critical section for sie64a
|
|
|
|
lgr %r14,%r9
|
|
|
|
slg %r14,BASED(.Lsie_critical_start)
|
|
|
|
clg %r14,BASED(.Lsie_critical_length)
|
|
|
|
jhe 0f
|
|
|
|
brasl %r14,.Lcleanup_sie
|
|
|
|
#endif
|
|
|
|
0: tmhh %r8,0x4000 # PER bit set in old PSW ?
|
|
|
|
jnz 1f # -> enabled, can't be a double fault
|
2011-12-27 10:27:15 +00:00
|
|
|
tm __LC_PGM_ILC+3,0x80 # check for per exception
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lpgm_svcper # -> single stepped svc
|
2015-06-22 15:26:40 +00:00
|
|
|
1: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
|
2013-04-24 08:20:43 +00:00
|
|
|
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
2015-06-22 15:26:40 +00:00
|
|
|
j 3f
|
2015-06-22 15:27:48 +00:00
|
|
|
2: LAST_BREAK %r14
|
|
|
|
UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r15,__LC_KERNEL_STACK
|
2012-07-31 09:03:04 +00:00
|
|
|
lg %r14,__TI_task(%r12)
|
2015-07-20 08:01:46 +00:00
|
|
|
aghi %r14,__TASK_thread # pointer to thread_struct
|
2012-07-31 09:03:04 +00:00
|
|
|
lghi %r13,__LC_PGM_TDB
|
|
|
|
tm __LC_PGM_ILC+2,0x02 # check for transaction abort
|
2015-06-22 15:26:40 +00:00
|
|
|
jz 3f
|
2012-07-31 09:03:04 +00:00
|
|
|
mvc __THREAD_trap_tdb(256,%r14),0(%r13)
|
2015-06-22 15:26:40 +00:00
|
|
|
3: la %r11,STACK_FRAME_OVERHEAD(%r15)
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r0,%r7,__PT_R0(%r11)
|
|
|
|
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
|
|
|
|
stmg %r8,%r9,__PT_PSW(%r11)
|
2011-12-27 10:27:18 +00:00
|
|
|
mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
|
|
|
|
mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
|
2014-04-15 10:55:07 +00:00
|
|
|
xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
|
2011-12-27 10:27:15 +00:00
|
|
|
stg %r10,__PT_ARGS(%r11)
|
|
|
|
tm __LC_PGM_ILC+3,0x80 # check for per exception
|
2015-06-22 15:26:40 +00:00
|
|
|
jz 4f
|
2011-12-27 10:27:15 +00:00
|
|
|
tmhh %r8,0x0001 # kernel per event ?
|
2014-12-03 16:00:08 +00:00
|
|
|
jz .Lpgm_kprobe
|
2014-04-15 10:55:07 +00:00
|
|
|
oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP
|
2012-07-31 09:03:04 +00:00
|
|
|
mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
|
2014-02-26 15:32:46 +00:00
|
|
|
mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE
|
|
|
|
mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
|
2015-06-22 15:26:40 +00:00
|
|
|
4: REENABLE_IRQS
|
2011-12-27 10:27:15 +00:00
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
2010-07-27 17:29:37 +00:00
|
|
|
larl %r1,pgm_check_table
|
2011-12-27 10:27:18 +00:00
|
|
|
llgh %r10,__PT_INT_CODE+2(%r11)
|
|
|
|
nill %r10,0x007f
|
2012-10-18 16:10:06 +00:00
|
|
|
sll %r10,2
|
2015-06-22 15:27:48 +00:00
|
|
|
je .Lpgm_return
|
2012-10-18 16:10:06 +00:00
|
|
|
lgf %r1,0(%r10,%r1) # load address of handler routine
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2010-07-27 17:29:37 +00:00
|
|
|
basr %r14,%r1 # branch to interrupt-handler
|
2015-06-22 15:27:48 +00:00
|
|
|
.Lpgm_return:
|
|
|
|
LOCKDEP_SYS_EXIT
|
|
|
|
tm __PT_PSW+1(%r11),0x01 # returning to user ?
|
|
|
|
jno .Lsysc_restore
|
|
|
|
j .Lsysc_tif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#
|
2011-12-27 10:27:15 +00:00
|
|
|
# PER event in supervisor state, must be kprobes
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lpgm_kprobe:
|
2011-12-27 10:27:15 +00:00
|
|
|
REENABLE_IRQS
|
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
|
|
|
brasl %r14,do_per_trap
|
2015-06-22 15:27:48 +00:00
|
|
|
j .Lpgm_return
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-09-20 13:58:39 +00:00
|
|
|
#
|
2011-12-27 10:27:15 +00:00
|
|
|
# single stepped system call
|
2006-09-20 13:58:39 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lpgm_svcper:
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r14,.Lsysc_per
|
2011-12-27 10:27:15 +00:00
|
|
|
stg %r14,__LC_RETURN_PSW+8
|
2014-04-15 10:55:07 +00:00
|
|
|
lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP
|
2014-12-03 16:00:08 +00:00
|
|
|
lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs
|
2006-09-20 13:58:39 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* IO interrupt handler routine
|
|
|
|
*/
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(io_int_handler)
|
2012-05-09 14:27:39 +00:00
|
|
|
STCK __LC_INT_CLOCK
|
2008-12-31 14:11:41 +00:00
|
|
|
stpt __LC_ASYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
|
|
|
|
lg %r10,__LC_LAST_BREAK
|
|
|
|
lg %r12,__LC_THREAD_INFO
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
larl %r13,cleanup_critical
|
2011-12-27 10:27:15 +00:00
|
|
|
lmg %r8,%r9,__LC_IO_OLD_PSW
|
2015-06-22 15:28:14 +00:00
|
|
|
SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r0,%r7,__PT_R0(%r11)
|
|
|
|
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
|
|
|
|
stmg %r8,%r9,__PT_PSW(%r11)
|
2013-06-17 12:54:02 +00:00
|
|
|
mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
|
2014-04-15 10:55:07 +00:00
|
|
|
xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
|
2006-07-03 07:24:46 +00:00
|
|
|
TRACE_IRQS_OFF
|
2011-12-27 10:27:15 +00:00
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_loop:
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2013-06-27 07:01:09 +00:00
|
|
|
lghi %r3,IO_INTERRUPT
|
|
|
|
tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
|
2014-12-03 16:00:08 +00:00
|
|
|
jz .Lio_call
|
2013-06-27 07:01:09 +00:00
|
|
|
lghi %r3,THIN_INTERRUPT
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_call:
|
2011-12-27 10:27:15 +00:00
|
|
|
brasl %r14,do_IRQ
|
2013-06-17 12:54:02 +00:00
|
|
|
tm __LC_MACHINE_FLAGS+6,0x10 # MACHINE_FLAG_LPAR
|
2014-12-03 16:00:08 +00:00
|
|
|
jz .Lio_return
|
2013-06-17 12:54:02 +00:00
|
|
|
tpi 0
|
2014-12-03 16:00:08 +00:00
|
|
|
jz .Lio_return
|
2013-06-17 12:54:02 +00:00
|
|
|
mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_loop
|
|
|
|
.Lio_return:
|
2010-05-17 08:00:02 +00:00
|
|
|
LOCKDEP_SYS_EXIT
|
|
|
|
TRACE_IRQS_ON
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_tif:
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_WORK
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lio_work # there is work to do (signals etc.)
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_WORK
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lio_work
|
|
|
|
.Lio_restore:
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r14,__LC_VDSO_PER_CPU
|
|
|
|
lmg %r0,%r10,__PT_R0(%r11)
|
|
|
|
mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
|
|
|
|
stpt __LC_EXIT_TIMER
|
|
|
|
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
|
|
|
lmg %r11,%r15,__PT_R11(%r11)
|
|
|
|
lpswe __LC_RETURN_PSW
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_done:
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-05-07 07:22:52 +00:00
|
|
|
#
|
2010-05-17 08:00:01 +00:00
|
|
|
# There is work todo, find out in which context we have been interrupted:
|
2014-04-15 10:55:07 +00:00
|
|
|
# 1) if we return to user space we can do all _TIF_WORK work
|
2010-05-17 08:00:01 +00:00
|
|
|
# 2) if we return to kernel code and kvm is enabled check if we need to
|
|
|
|
# modify the psw to leave SIE
|
|
|
|
# 3) if we return to kernel code and preemptive scheduling is enabled check
|
|
|
|
# the preemption counter and if it is zero call preempt_schedule_irq
|
|
|
|
# Before any work can be done, a switch to the kernel stack is required.
|
2008-05-07 07:22:52 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_work:
|
2011-12-27 10:27:15 +00:00
|
|
|
tm __PT_PSW+1(%r11),0x01 # returning to user ?
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lio_work_user # yes -> do resched & signal
|
2010-05-17 08:00:01 +00:00
|
|
|
#ifdef CONFIG_PREEMPT
|
2008-05-07 07:22:52 +00:00
|
|
|
# check for preemptive scheduling
|
2010-05-17 08:00:05 +00:00
|
|
|
icm %r0,15,__TI_precount(%r12)
|
2014-12-03 16:00:08 +00:00
|
|
|
jnz .Lio_restore # preemption is disabled
|
2010-05-17 08:00:02 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_NEED_RESCHED
|
2014-12-03 16:00:08 +00:00
|
|
|
jno .Lio_restore
|
2005-04-16 22:20:36 +00:00
|
|
|
# switch to kernel stack
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r1,__PT_R15(%r11)
|
|
|
|
aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
|
|
|
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
|
|
|
|
xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
|
|
|
|
la %r11,STACK_FRAME_OVERHEAD(%r1)
|
2005-04-16 22:20:36 +00:00
|
|
|
lgr %r15,%r1
|
2014-12-03 16:00:08 +00:00
|
|
|
# TRACE_IRQS_ON already done at .Lio_return, call
|
2010-05-17 08:00:02 +00:00
|
|
|
# TRACE_IRQS_OFF to keep things symmetrical
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
brasl %r14,preempt_schedule_irq
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
2010-05-17 08:00:02 +00:00
|
|
|
#else
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_restore
|
2010-05-17 08:00:02 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-05-17 08:00:01 +00:00
|
|
|
#
|
|
|
|
# Need to do work before returning to userspace, switch to kernel stack
|
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_work_user:
|
2005-04-16 22:20:36 +00:00
|
|
|
lg %r1,__LC_KERNEL_STACK
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
|
|
|
|
xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
|
|
|
|
la %r11,STACK_FRAME_OVERHEAD(%r1)
|
2005-04-16 22:20:36 +00:00
|
|
|
lgr %r15,%r1
|
2010-05-17 08:00:01 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
|
|
|
# One of the work bits is on. Find out which one.
|
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_work_tif:
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lio_mcck_pending
|
2010-05-17 08:00:05 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_NEED_RESCHED
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lio_reschedule
|
2010-05-17 08:00:05 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_SIGPENDING
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lio_sigpending
|
2010-05-17 08:00:05 +00:00
|
|
|
tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lio_notify_resume
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
jo .Lio_vxrs
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_ASCE
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lio_uaccess
|
|
|
|
j .Lio_return # beware of critical section cleanup
|
2008-05-07 07:22:53 +00:00
|
|
|
|
2005-06-25 21:55:30 +00:00
|
|
|
#
|
2014-04-15 10:55:07 +00:00
|
|
|
# _CIF_MCCK_PENDING is set, call handler
|
2005-06-25 21:55:30 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_mcck_pending:
|
|
|
|
# TRACE_IRQS_ON already done at .Lio_return
|
2007-07-27 10:29:18 +00:00
|
|
|
brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler
|
2010-05-17 08:00:02 +00:00
|
|
|
TRACE_IRQS_OFF
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
2005-06-25 21:55:30 +00:00
|
|
|
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
#
|
2014-04-15 10:55:07 +00:00
|
|
|
# _CIF_ASCE is set, load user space asce
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_uaccess:
|
2014-04-15 10:55:07 +00:00
|
|
|
ni __LC_CPU_FLAGS+7,255-_CIF_ASCE
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
s390/uaccess: rework uaccess code - fix locking issues
The current uaccess code uses a page table walk in some circumstances,
e.g. in case of the in atomic futex operations or if running on old
hardware which doesn't support the mvcos instruction.
However it turned out that the page table walk code does not correctly
lock page tables when accessing page table entries.
In other words: a different cpu may invalidate a page table entry while
the current cpu inspects the pte. This may lead to random data corruption.
Adding correct locking however isn't trivial for all uaccess operations.
Especially copy_in_user() is problematic since that requires to hold at
least two locks, but must be protected against ABBA deadlock when a
different cpu also performs a copy_in_user() operation.
So the solution is a different approach where we change address spaces:
User space runs in primary address mode, or access register mode within
vdso code, like it currently already does.
The kernel usually also runs in home space mode, however when accessing
user space the kernel switches to primary or secondary address mode if
the mvcos instruction is not available or if a compare-and-swap (futex)
instruction on a user space address is performed.
KVM however is special, since that requires the kernel to run in home
address space while implicitly accessing user space with the sie
instruction.
So we end up with:
User space:
- runs in primary or access register mode
- cr1 contains the user asce
- cr7 contains the user asce
- cr13 contains the kernel asce
Kernel space:
- runs in home space mode
- cr1 contains the user or kernel asce
-> the kernel asce is loaded when a uaccess requires primary or
secondary address mode
- cr7 contains the user or kernel asce, (changed with set_fs())
- cr13 contains the kernel asce
In case of uaccess the kernel changes to:
- primary space mode in case of a uaccess (copy_to_user) and uses
e.g. the mvcp instruction to access user space. However the kernel
will stay in home space mode if the mvcos instruction is available
- secondary space mode in case of futex atomic operations, so that the
instructions come from primary address space and data from secondary
space
In case of kvm the kernel runs in home space mode, but cr1 gets switched
to contain the gmap asce before the sie instruction gets executed. When
the sie instruction is finished cr1 will be switched back to contain the
user asce.
A context switch between two processes will always load the kernel asce
for the next process in cr1. So the first exit to user space is a bit
more expensive (one extra load control register instruction) than before,
however keeps the code rather simple.
In sum this means there is no need to perform any error prone page table
walks anymore when accessing user space.
The patch seems to be rather large, however it mainly removes the
the page table walk code and restores the previously deleted "standard"
uaccess code, with a couple of changes.
The uaccess without mvcos mode can be enforced with the "uaccess_primary"
kernel parameter.
Reported-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2014-03-21 09:42:25 +00:00
|
|
|
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
#
|
|
|
|
# CIF_FPU is set, restore floating-point controls and floating-point registers.
|
|
|
|
#
|
|
|
|
.Lio_vxrs:
|
|
|
|
larl %r14,.Lio_return
|
|
|
|
jg load_fpu_regs
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
|
|
|
# _TIF_NEED_RESCHED is set, call schedule
|
2006-09-28 14:56:37 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_reschedule:
|
|
|
|
# TRACE_IRQS_ON already done at .Lio_return
|
2011-12-27 10:27:15 +00:00
|
|
|
ssm __LC_SVC_NEW_PSW # reenable interrupts
|
2006-09-28 14:56:37 +00:00
|
|
|
brasl %r14,schedule # call scheduler
|
2011-12-27 10:27:15 +00:00
|
|
|
ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
|
2007-11-20 10:13:32 +00:00
|
|
|
TRACE_IRQS_OFF
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#
|
2008-04-30 07:53:08 +00:00
|
|
|
# _TIF_SIGPENDING or is set, call do_signal
|
2005-04-16 22:20:36 +00:00
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_sigpending:
|
|
|
|
# TRACE_IRQS_ON already done at .Lio_return
|
2011-12-27 10:27:15 +00:00
|
|
|
ssm __LC_SVC_NEW_PSW # reenable interrupts
|
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
|
|
|
brasl %r14,do_signal
|
|
|
|
ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
|
2007-11-20 10:13:32 +00:00
|
|
|
TRACE_IRQS_OFF
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-10-10 19:33:20 +00:00
|
|
|
#
|
|
|
|
# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
|
|
|
|
#
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lio_notify_resume:
|
|
|
|
# TRACE_IRQS_ON already done at .Lio_return
|
2011-12-27 10:27:15 +00:00
|
|
|
ssm __LC_SVC_NEW_PSW # reenable interrupts
|
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
|
|
|
brasl %r14,do_notify_resume
|
|
|
|
ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
|
2008-10-10 19:33:20 +00:00
|
|
|
TRACE_IRQS_OFF
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
2008-10-10 19:33:20 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* External interrupt handler routine
|
|
|
|
*/
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(ext_int_handler)
|
2012-05-09 14:27:39 +00:00
|
|
|
STCK __LC_INT_CLOCK
|
2008-12-31 14:11:41 +00:00
|
|
|
stpt __LC_ASYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
|
|
|
|
lg %r10,__LC_LAST_BREAK
|
|
|
|
lg %r12,__LC_THREAD_INFO
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
larl %r13,cleanup_critical
|
2011-12-27 10:27:15 +00:00
|
|
|
lmg %r8,%r9,__LC_EXT_OLD_PSW
|
2015-06-22 15:28:14 +00:00
|
|
|
SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r0,%r7,__PT_R0(%r11)
|
|
|
|
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
|
|
|
|
stmg %r8,%r9,__PT_PSW(%r11)
|
2013-06-17 12:54:02 +00:00
|
|
|
lghi %r1,__LC_EXT_PARAMS2
|
|
|
|
mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
|
|
|
|
mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
|
|
|
|
mvc __PT_INT_PARM_LONG(8,%r11),0(%r1)
|
2014-04-15 10:55:07 +00:00
|
|
|
xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
|
2006-07-03 07:24:46 +00:00
|
|
|
TRACE_IRQS_OFF
|
2012-05-09 14:27:35 +00:00
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
2011-12-27 10:27:15 +00:00
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2013-06-27 07:01:09 +00:00
|
|
|
lghi %r3,EXT_INTERRUPT
|
|
|
|
brasl %r14,do_IRQ
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lio_return
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-03-11 15:59:27 +00:00
|
|
|
/*
|
2014-12-03 16:00:08 +00:00
|
|
|
* Load idle PSW. The second "half" of this function is in .Lcleanup_idle.
|
2012-03-11 15:59:27 +00:00
|
|
|
*/
|
|
|
|
ENTRY(psw_idle)
|
2012-07-20 09:15:08 +00:00
|
|
|
stg %r3,__SF_EMPTY(%r15)
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r1,.Lpsw_idle_lpsw+4
|
2012-03-11 15:59:27 +00:00
|
|
|
stg %r1,__SF_EMPTY+8(%r15)
|
2015-09-18 14:41:36 +00:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
larl %r1,smp_cpu_mtid
|
|
|
|
llgf %r1,0(%r1)
|
|
|
|
ltgr %r1,%r1
|
|
|
|
jz .Lpsw_idle_stcctm
|
|
|
|
.insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
|
|
|
|
.Lpsw_idle_stcctm:
|
|
|
|
#endif
|
2012-07-20 09:15:08 +00:00
|
|
|
STCK __CLOCK_IDLE_ENTER(%r2)
|
|
|
|
stpt __TIMER_IDLE_ENTER(%r2)
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lpsw_idle_lpsw:
|
2012-03-11 15:59:27 +00:00
|
|
|
lpswe __SF_EMPTY(%r15)
|
|
|
|
br %r14
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lpsw_idle_end:
|
2012-03-11 15:59:27 +00:00
|
|
|
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
/* Store floating-point controls and floating-point or vector extension
|
|
|
|
* registers instead. A critical section cleanup assures that the registers
|
|
|
|
* are stored even if interrupted for some other work. The register %r2
|
|
|
|
* designates a struct fpu to store register contents. If the specified
|
|
|
|
* structure does not contain a register save area, the register store is
|
|
|
|
* omitted (see also comments in arch_dup_task_struct()).
|
|
|
|
*
|
|
|
|
* The CIF_FPU flag is set in any case. The CIF_FPU triggers a lazy restore
|
|
|
|
* of the register contents at system call or io return.
|
|
|
|
*/
|
|
|
|
ENTRY(save_fpu_regs)
|
2015-06-29 14:43:06 +00:00
|
|
|
lg %r2,__LC_CURRENT
|
|
|
|
aghi %r2,__TASK_thread
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
bor %r14
|
2015-06-29 14:43:06 +00:00
|
|
|
stfpc __THREAD_FPU_fpc(%r2)
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
.Lsave_fpu_regs_fpc_end:
|
2015-06-29 14:43:06 +00:00
|
|
|
lg %r3,__THREAD_FPU_regs(%r2)
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
ltgr %r3,%r3
|
|
|
|
jz .Lsave_fpu_regs_done # no save area -> set CIF_FPU
|
2015-06-29 14:43:06 +00:00
|
|
|
tm __THREAD_FPU_flags+3(%r2),FPU_USE_VX
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
jz .Lsave_fpu_regs_fp # no -> store FP regs
|
|
|
|
.Lsave_fpu_regs_vx_low:
|
|
|
|
VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3)
|
|
|
|
.Lsave_fpu_regs_vx_high:
|
|
|
|
VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3)
|
|
|
|
j .Lsave_fpu_regs_done # -> set CIF_FPU flag
|
|
|
|
.Lsave_fpu_regs_fp:
|
|
|
|
std 0,0(%r3)
|
|
|
|
std 1,8(%r3)
|
|
|
|
std 2,16(%r3)
|
|
|
|
std 3,24(%r3)
|
|
|
|
std 4,32(%r3)
|
|
|
|
std 5,40(%r3)
|
|
|
|
std 6,48(%r3)
|
|
|
|
std 7,56(%r3)
|
|
|
|
std 8,64(%r3)
|
|
|
|
std 9,72(%r3)
|
|
|
|
std 10,80(%r3)
|
|
|
|
std 11,88(%r3)
|
|
|
|
std 12,96(%r3)
|
|
|
|
std 13,104(%r3)
|
|
|
|
std 14,112(%r3)
|
|
|
|
std 15,120(%r3)
|
|
|
|
.Lsave_fpu_regs_done:
|
|
|
|
oi __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
br %r14
|
|
|
|
.Lsave_fpu_regs_end:
|
|
|
|
|
|
|
|
/* Load floating-point controls and floating-point or vector extension
|
|
|
|
* registers. A critical section cleanup assures that the register contents
|
|
|
|
* are loaded even if interrupted for some other work. Depending on the saved
|
|
|
|
* FP/VX state, the vector-enablement control, CR0.46, is either set or cleared.
|
|
|
|
*
|
|
|
|
* There are special calling conventions to fit into sysc and io return work:
|
|
|
|
* %r15: <kernel stack>
|
|
|
|
* The function requires:
|
|
|
|
* %r4 and __SF_EMPTY+32(%r15)
|
|
|
|
*/
|
|
|
|
load_fpu_regs:
|
2015-06-29 14:43:06 +00:00
|
|
|
lg %r4,__LC_CURRENT
|
|
|
|
aghi %r4,__TASK_thread
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
bnor %r14
|
2015-06-29 14:43:06 +00:00
|
|
|
lfpc __THREAD_FPU_fpc(%r4)
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
stctg %c0,%c0,__SF_EMPTY+32(%r15) # store CR0
|
2015-06-29 14:43:06 +00:00
|
|
|
tm __THREAD_FPU_flags+3(%r4),FPU_USE_VX # VX-enabled task ?
|
|
|
|
lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
jz .Lload_fpu_regs_fp_ctl # -> no VX, load FP regs
|
|
|
|
.Lload_fpu_regs_vx_ctl:
|
|
|
|
tm __SF_EMPTY+32+5(%r15),2 # test VX control
|
|
|
|
jo .Lload_fpu_regs_vx
|
|
|
|
oi __SF_EMPTY+32+5(%r15),2 # set VX control
|
|
|
|
lctlg %c0,%c0,__SF_EMPTY+32(%r15)
|
|
|
|
.Lload_fpu_regs_vx:
|
|
|
|
VLM %v0,%v15,0,%r4
|
|
|
|
.Lload_fpu_regs_vx_high:
|
|
|
|
VLM %v16,%v31,256,%r4
|
|
|
|
j .Lload_fpu_regs_done
|
|
|
|
.Lload_fpu_regs_fp_ctl:
|
|
|
|
tm __SF_EMPTY+32+5(%r15),2 # test VX control
|
|
|
|
jz .Lload_fpu_regs_fp
|
|
|
|
ni __SF_EMPTY+32+5(%r15),253 # clear VX control
|
|
|
|
lctlg %c0,%c0,__SF_EMPTY+32(%r15)
|
|
|
|
.Lload_fpu_regs_fp:
|
|
|
|
ld 0,0(%r4)
|
|
|
|
ld 1,8(%r4)
|
|
|
|
ld 2,16(%r4)
|
|
|
|
ld 3,24(%r4)
|
|
|
|
ld 4,32(%r4)
|
|
|
|
ld 5,40(%r4)
|
|
|
|
ld 6,48(%r4)
|
|
|
|
ld 7,56(%r4)
|
|
|
|
ld 8,64(%r4)
|
|
|
|
ld 9,72(%r4)
|
|
|
|
ld 10,80(%r4)
|
|
|
|
ld 11,88(%r4)
|
|
|
|
ld 12,96(%r4)
|
|
|
|
ld 13,104(%r4)
|
|
|
|
ld 14,112(%r4)
|
|
|
|
ld 15,120(%r4)
|
|
|
|
.Lload_fpu_regs_done:
|
|
|
|
ni __LC_CPU_FLAGS+7,255-_CIF_FPU
|
|
|
|
br %r14
|
|
|
|
.Lload_fpu_regs_end:
|
|
|
|
|
|
|
|
/* Test and set the vector enablement control in CR0.46 */
|
|
|
|
ENTRY(__ctl_set_vx)
|
|
|
|
stctg %c0,%c0,__SF_EMPTY(%r15)
|
|
|
|
tm __SF_EMPTY+5(%r15),2
|
|
|
|
bor %r14
|
|
|
|
oi __SF_EMPTY+5(%r15),2
|
|
|
|
lctlg %c0,%c0,__SF_EMPTY(%r15)
|
|
|
|
br %r14
|
|
|
|
.L__ctl_set_vx_end:
|
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.L__critical_end:
|
2005-09-03 22:57:56 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Machine check handler routines
|
|
|
|
*/
|
2011-07-24 08:48:19 +00:00
|
|
|
ENTRY(mcck_int_handler)
|
2012-05-09 14:27:39 +00:00
|
|
|
STCK __LC_MCCK_CLOCK
|
2005-06-25 21:55:30 +00:00
|
|
|
la %r1,4095 # revalidate r1
|
|
|
|
spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer
|
2006-09-28 14:56:37 +00:00
|
|
|
lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r10,__LC_LAST_BREAK
|
|
|
|
lg %r12,__LC_THREAD_INFO
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
larl %r13,cleanup_critical
|
2011-12-27 10:27:15 +00:00
|
|
|
lmg %r8,%r9,__LC_MCK_OLD_PSW
|
2006-09-28 14:56:37 +00:00
|
|
|
tm __LC_MCCK_CODE,0x80 # system damage?
|
2014-12-03 16:00:08 +00:00
|
|
|
jo .Lmcck_panic # yes -> rest of mcck code invalid
|
2011-12-27 10:27:15 +00:00
|
|
|
lghi %r14,__LC_CPU_TIMER_SAVE_AREA
|
|
|
|
mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
|
2006-06-29 12:58:05 +00:00
|
|
|
tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
|
2011-12-27 10:27:15 +00:00
|
|
|
jo 3f
|
2006-06-29 12:58:05 +00:00
|
|
|
la %r14,__LC_SYNC_ENTER_TIMER
|
|
|
|
clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
|
|
|
|
jl 0f
|
|
|
|
la %r14,__LC_ASYNC_ENTER_TIMER
|
|
|
|
0: clc 0(8,%r14),__LC_EXIT_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
jl 1f
|
2006-06-29 12:58:05 +00:00
|
|
|
la %r14,__LC_EXIT_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
|
|
|
|
jl 2f
|
2006-06-29 12:58:05 +00:00
|
|
|
la %r14,__LC_LAST_UPDATE_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
2: spt 0(%r14)
|
2010-05-17 08:00:03 +00:00
|
|
|
mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
|
2011-12-27 10:27:15 +00:00
|
|
|
3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
|
2014-12-03 16:00:08 +00:00
|
|
|
jno .Lmcck_panic # no -> skip cleanup critical
|
2015-06-22 15:28:14 +00:00
|
|
|
SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lmcck_skip:
|
2013-02-28 15:28:41 +00:00
|
|
|
lghi %r14,__LC_GPREGS_SAVE_AREA+64
|
|
|
|
stmg %r0,%r7,__PT_R0(%r11)
|
|
|
|
mvc __PT_R8(64,%r11),0(%r14)
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r8,%r9,__PT_PSW(%r11)
|
2014-04-15 10:55:07 +00:00
|
|
|
xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
|
2011-12-27 10:27:15 +00:00
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2005-06-25 21:55:30 +00:00
|
|
|
brasl %r14,s390_do_machine_check
|
2011-12-27 10:27:15 +00:00
|
|
|
tm __PT_PSW+1(%r11),0x01 # returning to user ?
|
2014-12-03 16:00:08 +00:00
|
|
|
jno .Lmcck_return
|
2005-06-25 21:55:30 +00:00
|
|
|
lg %r1,__LC_KERNEL_STACK # switch to kernel stack
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
|
|
|
|
xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
|
|
|
|
la %r11,STACK_FRAME_OVERHEAD(%r1)
|
2005-06-25 21:55:30 +00:00
|
|
|
lgr %r15,%r1
|
2011-12-27 10:27:15 +00:00
|
|
|
ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
|
2014-04-15 10:55:07 +00:00
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING
|
2014-12-03 16:00:08 +00:00
|
|
|
jno .Lmcck_return
|
2006-07-03 07:24:46 +00:00
|
|
|
TRACE_IRQS_OFF
|
2005-06-25 21:55:30 +00:00
|
|
|
brasl %r14,s390_handle_mcck
|
2006-07-03 07:24:46 +00:00
|
|
|
TRACE_IRQS_ON
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lmcck_return:
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r14,__LC_VDSO_PER_CPU
|
|
|
|
lmg %r0,%r10,__PT_R0(%r11)
|
|
|
|
mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
|
2006-06-29 12:58:05 +00:00
|
|
|
tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
|
|
|
|
jno 0f
|
|
|
|
stpt __LC_EXIT_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
|
|
|
0: lmg %r11,%r15,__PT_R11(%r11)
|
|
|
|
lpswe __LC_RETURN_MCCK_PSW
|
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lmcck_panic:
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r15,__LC_PANIC_STACK
|
2015-06-22 15:28:14 +00:00
|
|
|
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
2014-12-03 16:00:08 +00:00
|
|
|
j .Lmcck_skip
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-08-03 14:44:19 +00:00
|
|
|
#
|
|
|
|
# PSW restart interrupt handler
|
|
|
|
#
|
2012-03-11 15:59:26 +00:00
|
|
|
ENTRY(restart_int_handler)
|
2011-12-27 10:27:15 +00:00
|
|
|
stg %r15,__LC_SAVE_AREA_RESTART
|
2012-03-11 15:59:26 +00:00
|
|
|
lg %r15,__LC_RESTART_STACK
|
2011-12-27 10:27:15 +00:00
|
|
|
aghi %r15,-__PT_SIZE # create pt_regs on stack
|
2012-03-11 15:59:26 +00:00
|
|
|
xc 0(__PT_SIZE,%r15),0(%r15)
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r0,%r14,__PT_R0(%r15)
|
|
|
|
mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
|
|
|
|
mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
|
2012-03-11 15:59:26 +00:00
|
|
|
aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
|
|
|
|
xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
|
2012-06-05 07:59:52 +00:00
|
|
|
lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
|
|
|
|
lg %r2,__LC_RESTART_DATA
|
|
|
|
lg %r3,__LC_RESTART_SOURCE
|
2012-03-11 15:59:26 +00:00
|
|
|
ltgr %r3,%r3 # test source cpu address
|
|
|
|
jm 1f # negative -> skip source stop
|
2012-06-04 13:05:43 +00:00
|
|
|
0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
|
2012-03-11 15:59:26 +00:00
|
|
|
brc 10,0b # wait for status stored
|
|
|
|
1: basr %r14,%r1 # call function
|
|
|
|
stap __SF_EMPTY(%r15) # store cpu address
|
|
|
|
llgh %r3,__SF_EMPTY(%r15)
|
2012-06-04 13:05:43 +00:00
|
|
|
2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
|
2012-03-11 15:59:26 +00:00
|
|
|
brc 2,2b
|
|
|
|
3: j 3b
|
2011-08-03 14:44:19 +00:00
|
|
|
|
2011-01-05 11:47:25 +00:00
|
|
|
.section .kprobes.text, "ax"
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_CHECK_STACK
|
|
|
|
/*
|
|
|
|
* The synchronous or the asynchronous stack overflowed. We are dead.
|
|
|
|
* No need to properly save the registers, we are going to panic anyway.
|
|
|
|
* Setup a pt_regs so that show_trace can provide a good call trace.
|
|
|
|
*/
|
|
|
|
stack_overflow:
|
2013-04-24 08:20:43 +00:00
|
|
|
lg %r15,__LC_PANIC_STACK # change to panic stack
|
|
|
|
la %r11,STACK_FRAME_OVERHEAD(%r15)
|
2011-12-27 10:27:15 +00:00
|
|
|
stmg %r0,%r7,__PT_R0(%r11)
|
|
|
|
stmg %r8,%r9,__PT_PSW(%r11)
|
|
|
|
mvc __PT_R8(64,%r11),0(%r14)
|
|
|
|
stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
|
|
|
|
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
|
|
|
|
lgr %r2,%r11 # pass pointer to pt_regs
|
2005-04-16 22:20:36 +00:00
|
|
|
jg kernel_stack_overflow
|
|
|
|
#endif
|
|
|
|
|
|
|
|
cleanup_critical:
|
2015-06-22 15:26:40 +00:00
|
|
|
#if IS_ENABLED(CONFIG_KVM)
|
|
|
|
clg %r9,BASED(.Lcleanup_table_sie) # .Lsie_gmap
|
|
|
|
jl 0f
|
|
|
|
clg %r9,BASED(.Lcleanup_table_sie+8)# .Lsie_done
|
|
|
|
jl .Lcleanup_sie
|
|
|
|
#endif
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_table) # system_call
|
2005-04-16 22:20:36 +00:00
|
|
|
jl 0f
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_table+8) # .Lsysc_do_svc
|
|
|
|
jl .Lcleanup_system_call
|
|
|
|
clg %r9,BASED(.Lcleanup_table+16) # .Lsysc_tif
|
2005-04-16 22:20:36 +00:00
|
|
|
jl 0f
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_table+24) # .Lsysc_restore
|
|
|
|
jl .Lcleanup_sysc_tif
|
|
|
|
clg %r9,BASED(.Lcleanup_table+32) # .Lsysc_done
|
|
|
|
jl .Lcleanup_sysc_restore
|
|
|
|
clg %r9,BASED(.Lcleanup_table+40) # .Lio_tif
|
2006-06-29 12:58:05 +00:00
|
|
|
jl 0f
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_table+48) # .Lio_restore
|
|
|
|
jl .Lcleanup_io_tif
|
|
|
|
clg %r9,BASED(.Lcleanup_table+56) # .Lio_done
|
|
|
|
jl .Lcleanup_io_restore
|
|
|
|
clg %r9,BASED(.Lcleanup_table+64) # psw_idle
|
2012-03-11 15:59:27 +00:00
|
|
|
jl 0f
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_table+72) # .Lpsw_idle_end
|
|
|
|
jl .Lcleanup_idle
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_table+80) # save_fpu_regs
|
|
|
|
jl 0f
|
|
|
|
clg %r9,BASED(.Lcleanup_table+88) # .Lsave_fpu_regs_end
|
|
|
|
jl .Lcleanup_save_fpu_regs
|
|
|
|
clg %r9,BASED(.Lcleanup_table+96) # load_fpu_regs
|
|
|
|
jl 0f
|
|
|
|
clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end
|
|
|
|
jl .Lcleanup_load_fpu_regs
|
|
|
|
clg %r9,BASED(.Lcleanup_table+112) # __ctl_set_vx
|
|
|
|
jl 0f
|
|
|
|
clg %r9,BASED(.Lcleanup_table+120) # .L__ctl_set_vx_end
|
|
|
|
jl .Lcleanup___ctl_set_vx
|
2011-12-27 10:27:15 +00:00
|
|
|
0: br %r14
|
|
|
|
|
2015-06-22 15:26:40 +00:00
|
|
|
.align 8
|
|
|
|
.Lcleanup_table:
|
|
|
|
.quad system_call
|
|
|
|
.quad .Lsysc_do_svc
|
|
|
|
.quad .Lsysc_tif
|
|
|
|
.quad .Lsysc_restore
|
|
|
|
.quad .Lsysc_done
|
|
|
|
.quad .Lio_tif
|
|
|
|
.quad .Lio_restore
|
|
|
|
.quad .Lio_done
|
|
|
|
.quad psw_idle
|
|
|
|
.quad .Lpsw_idle_end
|
|
|
|
.quad save_fpu_regs
|
|
|
|
.quad .Lsave_fpu_regs_end
|
|
|
|
.quad load_fpu_regs
|
|
|
|
.quad .Lload_fpu_regs_end
|
|
|
|
.quad __ctl_set_vx
|
|
|
|
.quad .L__ctl_set_vx_end
|
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_KVM)
|
|
|
|
.Lcleanup_table_sie:
|
|
|
|
.quad .Lsie_gmap
|
|
|
|
.quad .Lsie_done
|
|
|
|
|
|
|
|
.Lcleanup_sie:
|
|
|
|
lg %r9,__SF_EMPTY(%r15) # get control block pointer
|
|
|
|
tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_LPP
|
|
|
|
jz 0f
|
|
|
|
.insn s,0xb2800000,__SF_EMPTY+16(%r15)# set host id
|
|
|
|
0: ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
|
|
|
|
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
|
|
|
larl %r9,sie_exit # skip forward to sie_exit
|
|
|
|
br %r14
|
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_system_call:
|
2011-12-27 10:27:15 +00:00
|
|
|
# check if stpt has been executed
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_system_call_insn)
|
2005-04-16 22:20:36 +00:00
|
|
|
jh 0f
|
|
|
|
mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
cghi %r11,__LC_SAVE_AREA_ASYNC
|
2010-05-17 08:00:03 +00:00
|
|
|
je 0f
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
|
|
|
|
0: # check if stmg has been executed
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_system_call_insn+8)
|
2005-04-16 22:20:36 +00:00
|
|
|
jh 0f
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc __LC_SAVE_AREA_SYNC(64),0(%r11)
|
|
|
|
0: # check if base register setup + TIF bit load has been done
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_system_call_insn+16)
|
2011-12-27 10:27:15 +00:00
|
|
|
jhe 0f
|
|
|
|
# set up saved registers r10 and r12
|
|
|
|
stg %r10,16(%r11) # r10 last break
|
|
|
|
stg %r12,32(%r11) # r12 thread-info pointer
|
|
|
|
0: # check if the user time update has been done
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_system_call_insn+24)
|
2011-12-27 10:27:15 +00:00
|
|
|
jh 0f
|
|
|
|
lg %r15,__LC_EXIT_TIMER
|
|
|
|
slg %r15,__LC_SYNC_ENTER_TIMER
|
|
|
|
alg %r15,__LC_USER_TIMER
|
|
|
|
stg %r15,__LC_USER_TIMER
|
|
|
|
0: # check if the system time update has been done
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_system_call_insn+32)
|
2011-12-27 10:27:15 +00:00
|
|
|
jh 0f
|
|
|
|
lg %r15,__LC_LAST_UPDATE_TIMER
|
|
|
|
slg %r15,__LC_EXIT_TIMER
|
|
|
|
alg %r15,__LC_SYSTEM_TIMER
|
|
|
|
stg %r15,__LC_SYSTEM_TIMER
|
|
|
|
0: # update accounting time stamp
|
2005-04-16 22:20:36 +00:00
|
|
|
mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
|
2011-12-27 10:27:15 +00:00
|
|
|
# do LAST_BREAK
|
|
|
|
lg %r9,16(%r11)
|
|
|
|
srag %r9,%r9,23
|
2010-05-17 08:00:05 +00:00
|
|
|
jz 0f
|
2011-12-27 10:27:15 +00:00
|
|
|
mvc __TI_last_break(8,%r12),16(%r11)
|
|
|
|
0: # set up saved register r11
|
|
|
|
lg %r15,__LC_KERNEL_STACK
|
2013-04-24 08:20:43 +00:00
|
|
|
la %r9,STACK_FRAME_OVERHEAD(%r15)
|
|
|
|
stg %r9,24(%r11) # r11 pt_regs pointer
|
2011-12-27 10:27:15 +00:00
|
|
|
# fill pt_regs
|
2013-04-24 08:20:43 +00:00
|
|
|
mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
|
|
|
|
stmg %r0,%r7,__PT_R0(%r9)
|
|
|
|
mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
|
|
|
|
mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
|
2014-04-15 10:55:07 +00:00
|
|
|
xc __PT_FLAGS(8,%r9),__PT_FLAGS(%r9)
|
|
|
|
mvi __PT_FLAGS+7(%r9),_PIF_SYSCALL
|
2011-12-27 10:27:15 +00:00
|
|
|
# setup saved register r15
|
|
|
|
stg %r15,56(%r11) # r15 stack pointer
|
|
|
|
# set new psw address and exit
|
2014-12-03 16:00:08 +00:00
|
|
|
larl %r9,.Lsysc_do_svc
|
2005-04-16 22:20:36 +00:00
|
|
|
br %r14
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_system_call_insn:
|
2006-09-28 14:56:37 +00:00
|
|
|
.quad system_call
|
2014-12-03 16:00:08 +00:00
|
|
|
.quad .Lsysc_stmg
|
|
|
|
.quad .Lsysc_per
|
2015-06-22 15:27:48 +00:00
|
|
|
.quad .Lsysc_vtime+36
|
2014-12-03 16:00:08 +00:00
|
|
|
.quad .Lsysc_vtime+42
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_sysc_tif:
|
|
|
|
larl %r9,.Lsysc_tif
|
2005-04-16 22:20:36 +00:00
|
|
|
br %r14
|
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_sysc_restore:
|
|
|
|
clg %r9,BASED(.Lcleanup_sysc_restore_insn)
|
2010-05-17 08:00:03 +00:00
|
|
|
je 0f
|
2011-12-27 10:27:15 +00:00
|
|
|
lg %r9,24(%r11) # get saved pointer to pt_regs
|
|
|
|
mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
|
|
|
|
mvc 0(64,%r11),__PT_R8(%r9)
|
|
|
|
lmg %r0,%r7,__PT_R0(%r9)
|
|
|
|
0: lmg %r8,%r9,__LC_RETURN_PSW
|
2005-04-16 22:20:36 +00:00
|
|
|
br %r14
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_sysc_restore_insn:
|
|
|
|
.quad .Lsysc_done - 4
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_io_tif:
|
|
|
|
larl %r9,.Lio_tif
|
2010-04-09 11:43:00 +00:00
|
|
|
br %r14
|
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_io_restore:
|
|
|
|
clg %r9,BASED(.Lcleanup_io_restore_insn)
|
2011-12-27 10:27:15 +00:00
|
|
|
je 0f
|
|
|
|
lg %r9,24(%r11) # get saved r11 pointer to pt_regs
|
|
|
|
mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
|
|
|
|
mvc 0(64,%r11),__PT_R8(%r9)
|
|
|
|
lmg %r0,%r7,__PT_R0(%r9)
|
|
|
|
0: lmg %r8,%r9,__LC_RETURN_PSW
|
2005-09-03 22:57:56 +00:00
|
|
|
br %r14
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_io_restore_insn:
|
|
|
|
.quad .Lio_done - 4
|
2005-09-03 22:57:56 +00:00
|
|
|
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_idle:
|
2012-03-11 15:59:27 +00:00
|
|
|
# copy interrupt clock & cpu timer
|
2012-07-20 09:15:08 +00:00
|
|
|
mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
|
|
|
|
mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
|
2012-03-11 15:59:27 +00:00
|
|
|
cghi %r11,__LC_SAVE_AREA_ASYNC
|
|
|
|
je 0f
|
2012-07-20 09:15:08 +00:00
|
|
|
mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
|
|
|
|
mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
|
2012-03-11 15:59:27 +00:00
|
|
|
0: # check if stck & stpt have been executed
|
2014-12-03 16:00:08 +00:00
|
|
|
clg %r9,BASED(.Lcleanup_idle_insn)
|
2012-03-11 15:59:27 +00:00
|
|
|
jhe 1f
|
2012-07-20 09:15:08 +00:00
|
|
|
mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
|
|
|
|
mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
|
2015-09-18 14:41:36 +00:00
|
|
|
1: # calculate idle cycles
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
clg %r9,BASED(.Lcleanup_idle_insn)
|
|
|
|
jl 3f
|
|
|
|
larl %r1,smp_cpu_mtid
|
|
|
|
llgf %r1,0(%r1)
|
|
|
|
ltgr %r1,%r1
|
|
|
|
jz 3f
|
|
|
|
.insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
|
|
|
|
larl %r3,mt_cycles
|
|
|
|
ag %r3,__LC_PERCPU_OFFSET
|
|
|
|
la %r4,__SF_EMPTY+16(%r15)
|
|
|
|
2: lg %r0,0(%r3)
|
|
|
|
slg %r0,0(%r4)
|
|
|
|
alg %r0,64(%r4)
|
|
|
|
stg %r0,0(%r3)
|
|
|
|
la %r3,8(%r3)
|
|
|
|
la %r4,8(%r4)
|
|
|
|
brct %r1,2b
|
|
|
|
#endif
|
|
|
|
3: # account system time going idle
|
2012-03-11 15:59:27 +00:00
|
|
|
lg %r9,__LC_STEAL_TIMER
|
2012-07-20 09:15:08 +00:00
|
|
|
alg %r9,__CLOCK_IDLE_ENTER(%r2)
|
2012-03-11 15:59:27 +00:00
|
|
|
slg %r9,__LC_LAST_UPDATE_CLOCK
|
|
|
|
stg %r9,__LC_STEAL_TIMER
|
2012-07-20 09:15:08 +00:00
|
|
|
mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
|
2012-03-11 15:59:27 +00:00
|
|
|
lg %r9,__LC_SYSTEM_TIMER
|
|
|
|
alg %r9,__LC_LAST_UPDATE_TIMER
|
2012-07-20 09:15:08 +00:00
|
|
|
slg %r9,__TIMER_IDLE_ENTER(%r2)
|
2012-03-11 15:59:27 +00:00
|
|
|
stg %r9,__LC_SYSTEM_TIMER
|
2012-07-20 09:15:08 +00:00
|
|
|
mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
|
2012-03-11 15:59:27 +00:00
|
|
|
# prepare return psw
|
2013-08-23 12:45:58 +00:00
|
|
|
nihh %r8,0xfcfd # clear irq & wait state bits
|
2012-03-11 15:59:27 +00:00
|
|
|
lg %r9,48(%r11) # return from psw_idle
|
|
|
|
br %r14
|
2014-12-03 16:00:08 +00:00
|
|
|
.Lcleanup_idle_insn:
|
|
|
|
.quad .Lpsw_idle_lpsw
|
2012-03-11 15:59:27 +00:00
|
|
|
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
.Lcleanup_save_fpu_regs:
|
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
bor %r14
|
|
|
|
clg %r9,BASED(.Lcleanup_save_fpu_regs_done)
|
|
|
|
jhe 5f
|
|
|
|
clg %r9,BASED(.Lcleanup_save_fpu_regs_fp)
|
|
|
|
jhe 4f
|
|
|
|
clg %r9,BASED(.Lcleanup_save_fpu_regs_vx_high)
|
|
|
|
jhe 3f
|
|
|
|
clg %r9,BASED(.Lcleanup_save_fpu_regs_vx_low)
|
|
|
|
jhe 2f
|
|
|
|
clg %r9,BASED(.Lcleanup_save_fpu_fpc_end)
|
|
|
|
jhe 1f
|
2015-06-29 14:43:06 +00:00
|
|
|
lg %r2,__LC_CURRENT
|
2015-09-09 11:15:00 +00:00
|
|
|
aghi %r2,__TASK_thread
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
0: # Store floating-point controls
|
2015-06-29 14:43:06 +00:00
|
|
|
stfpc __THREAD_FPU_fpc(%r2)
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
1: # Load register save area and check if VX is active
|
2015-06-29 14:43:06 +00:00
|
|
|
lg %r3,__THREAD_FPU_regs(%r2)
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
ltgr %r3,%r3
|
|
|
|
jz 5f # no save area -> set CIF_FPU
|
2015-06-29 14:43:06 +00:00
|
|
|
tm __THREAD_FPU_flags+3(%r2),FPU_USE_VX
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
jz 4f # no VX -> store FP regs
|
|
|
|
2: # Store vector registers (V0-V15)
|
|
|
|
VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3)
|
|
|
|
3: # Store vector registers (V16-V31)
|
|
|
|
VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3)
|
|
|
|
j 5f # -> done, set CIF_FPU flag
|
|
|
|
4: # Store floating-point registers
|
|
|
|
std 0,0(%r3)
|
|
|
|
std 1,8(%r3)
|
|
|
|
std 2,16(%r3)
|
|
|
|
std 3,24(%r3)
|
|
|
|
std 4,32(%r3)
|
|
|
|
std 5,40(%r3)
|
|
|
|
std 6,48(%r3)
|
|
|
|
std 7,56(%r3)
|
|
|
|
std 8,64(%r3)
|
|
|
|
std 9,72(%r3)
|
|
|
|
std 10,80(%r3)
|
|
|
|
std 11,88(%r3)
|
|
|
|
std 12,96(%r3)
|
|
|
|
std 13,104(%r3)
|
|
|
|
std 14,112(%r3)
|
|
|
|
std 15,120(%r3)
|
|
|
|
5: # Set CIF_FPU flag
|
|
|
|
oi __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
lg %r9,48(%r11) # return from save_fpu_regs
|
|
|
|
br %r14
|
|
|
|
.Lcleanup_save_fpu_fpc_end:
|
|
|
|
.quad .Lsave_fpu_regs_fpc_end
|
|
|
|
.Lcleanup_save_fpu_regs_vx_low:
|
|
|
|
.quad .Lsave_fpu_regs_vx_low
|
|
|
|
.Lcleanup_save_fpu_regs_vx_high:
|
|
|
|
.quad .Lsave_fpu_regs_vx_high
|
|
|
|
.Lcleanup_save_fpu_regs_fp:
|
|
|
|
.quad .Lsave_fpu_regs_fp
|
|
|
|
.Lcleanup_save_fpu_regs_done:
|
|
|
|
.quad .Lsave_fpu_regs_done
|
|
|
|
|
|
|
|
.Lcleanup_load_fpu_regs:
|
|
|
|
tm __LC_CPU_FLAGS+7,_CIF_FPU
|
|
|
|
bnor %r14
|
|
|
|
clg %r9,BASED(.Lcleanup_load_fpu_regs_done)
|
|
|
|
jhe 1f
|
|
|
|
clg %r9,BASED(.Lcleanup_load_fpu_regs_fp)
|
|
|
|
jhe 2f
|
|
|
|
clg %r9,BASED(.Lcleanup_load_fpu_regs_fp_ctl)
|
|
|
|
jhe 3f
|
|
|
|
clg %r9,BASED(.Lcleanup_load_fpu_regs_vx_high)
|
|
|
|
jhe 4f
|
|
|
|
clg %r9,BASED(.Lcleanup_load_fpu_regs_vx)
|
|
|
|
jhe 5f
|
|
|
|
clg %r9,BASED(.Lcleanup_load_fpu_regs_vx_ctl)
|
|
|
|
jhe 6f
|
2015-06-29 14:43:06 +00:00
|
|
|
lg %r4,__LC_CURRENT
|
2015-09-09 11:15:00 +00:00
|
|
|
aghi %r4,__TASK_thread
|
2015-06-29 14:43:06 +00:00
|
|
|
lfpc __THREAD_FPU_fpc(%r4)
|
|
|
|
tm __THREAD_FPU_flags+3(%r4),FPU_USE_VX # VX-enabled task ?
|
|
|
|
lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
|
s390/kernel: lazy restore fpu registers
Improve the save and restore behavior of FPU register contents to use the
vector extension within the kernel.
The kernel does not use floating-point or vector registers and, therefore,
saving and restoring the FPU register contents are performed for handling
signals or switching processes only. To prepare for using vector
instructions and vector registers within the kernel, enhance the save
behavior and implement a lazy restore at return to user space from a
system call or interrupt.
To implement the lazy restore, the save_fpu_regs() sets a CPU information
flag, CIF_FPU, to indicate that the FPU registers must be restored.
Saving and setting CIF_FPU is performed in an atomic fashion to be
interrupt-safe. When the kernel wants to use the vector extension or
wants to change the FPU register state for a task during signal handling,
the save_fpu_regs() must be called first. The CIF_FPU flag is also set at
process switch. At return to user space, the FPU state is restored. In
particular, the FPU state includes the floating-point or vector register
contents, as well as, vector-enablement and floating-point control. The
FPU state restore and clearing CIF_FPU is also performed in an atomic
fashion.
For KVM, the restore of the FPU register state is performed when restoring
the general-purpose guest registers before the SIE instructions is started.
Because the path towards the SIE instruction is interruptible, the CIF_FPU
flag must be checked again right before going into SIE. If set, the guest
registers must be reloaded again by re-entering the outer SIE loop. This
is the same behavior as if the SIE critical section is interrupted.
Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2015-06-10 10:53:42 +00:00
|
|
|
jz 3f # -> no VX, load FP regs
|
|
|
|
6: # Set VX-enablement control
|
|
|
|
stctg %c0,%c0,__SF_EMPTY+32(%r15) # store CR0
|
|
|
|
tm __SF_EMPTY+32+5(%r15),2 # test VX control
|
|
|
|
jo 5f
|
|
|
|
oi __SF_EMPTY+32+5(%r15),2 # set VX control
|
|
|
|
lctlg %c0,%c0,__SF_EMPTY+32(%r15)
|
|
|
|
5: # Load V0 ..V15 registers
|
|
|
|
VLM %v0,%v15,0,%r4
|
|
|
|
4: # Load V16..V31 registers
|
|
|
|
VLM %v16,%v31,256,%r4
|
|
|
|
j 1f
|
|
|
|
3: # Clear VX-enablement control for FP
|
|
|
|
stctg %c0,%c0,__SF_EMPTY+32(%r15) # store CR0
|
|
|
|
tm __SF_EMPTY+32+5(%r15),2 # test VX control
|
|
|
|
jz 2f
|
|
|
|
ni __SF_EMPTY+32+5(%r15),253 # clear VX control
|
|
|
|
lctlg %c0,%c0,__SF_EMPTY+32(%r15)
|
|
|
|
2: # Load floating-point registers
|
|
|
|
ld 0,0(%r4)
|
|
|
|
ld 1,8(%r4)
|
|
|
|
ld 2,16(%r4)
|
|
|
|
ld 3,24(%r4)
|
|
|
|
ld 4,32(%r4)
|
|
|
|
ld 5,40(%r4)
|
|
|
|
ld 6,48(%r4)
|
|
|
|
ld 7,56(%r4)
|
|
|
|
ld 8,64(%r4)
|
|
|
|
ld 9,72(%r4)
|
|
|
|
ld 10,80(%r4)
|
|
|
|
ld 11,88(%r4)
|
|
|
|
ld 12,96(%r4)
|
|
|
|
ld 13,104(%r4)
|
|
|
|
ld 14,112(%r4)
|
|
|
|
ld 15,120(%r4)
|
|
|
|
1: # Clear CIF_FPU bit
|
|
|
|
ni __LC_CPU_FLAGS+7,255-_CIF_FPU
|
|
|
|
lg %r9,48(%r11) # return from load_fpu_regs
|
|
|
|
br %r14
|
|
|
|
.Lcleanup_load_fpu_regs_vx_ctl:
|
|
|
|
.quad .Lload_fpu_regs_vx_ctl
|
|
|
|
.Lcleanup_load_fpu_regs_vx:
|
|
|
|
.quad .Lload_fpu_regs_vx
|
|
|
|
.Lcleanup_load_fpu_regs_vx_high:
|
|
|
|
.quad .Lload_fpu_regs_vx_high
|
|
|
|
.Lcleanup_load_fpu_regs_fp_ctl:
|
|
|
|
.quad .Lload_fpu_regs_fp_ctl
|
|
|
|
.Lcleanup_load_fpu_regs_fp:
|
|
|
|
.quad .Lload_fpu_regs_fp
|
|
|
|
.Lcleanup_load_fpu_regs_done:
|
|
|
|
.quad .Lload_fpu_regs_done
|
|
|
|
|
|
|
|
.Lcleanup___ctl_set_vx:
|
|
|
|
stctg %c0,%c0,__SF_EMPTY(%r15)
|
|
|
|
tm __SF_EMPTY+5(%r15),2
|
|
|
|
bor %r14
|
|
|
|
oi __SF_EMPTY+5(%r15),2
|
|
|
|
lctlg %c0,%c0,__SF_EMPTY(%r15)
|
|
|
|
lg %r9,48(%r11) # return from __ctl_set_vx
|
|
|
|
br %r14
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Integer constants
|
|
|
|
*/
|
2011-12-27 10:27:15 +00:00
|
|
|
.align 8
|
2005-04-16 22:20:36 +00:00
|
|
|
.Lcritical_start:
|
2014-12-03 16:00:08 +00:00
|
|
|
.quad .L__critical_start
|
2011-12-27 10:27:15 +00:00
|
|
|
.Lcritical_length:
|
2014-12-03 16:00:08 +00:00
|
|
|
.quad .L__critical_end - .L__critical_start
|
2013-11-01 09:08:20 +00:00
|
|
|
#if IS_ENABLED(CONFIG_KVM)
|
2015-06-22 15:26:40 +00:00
|
|
|
.Lsie_critical_start:
|
2014-12-03 16:00:08 +00:00
|
|
|
.quad .Lsie_gmap
|
2013-05-17 12:41:37 +00:00
|
|
|
.Lsie_critical_length:
|
2014-12-03 16:00:08 +00:00
|
|
|
.quad .Lsie_done - .Lsie_gmap
|
2011-07-24 08:48:18 +00:00
|
|
|
#endif
|
|
|
|
|
2015-02-13 13:44:29 +00:00
|
|
|
.section .rodata, "a"
|
|
|
|
#define SYSCALL(esame,emu) .long esame
|
2009-06-12 08:26:47 +00:00
|
|
|
.globl sys_call_table
|
2005-04-16 22:20:36 +00:00
|
|
|
sys_call_table:
|
|
|
|
#include "syscalls.S"
|
|
|
|
#undef SYSCALL
|
|
|
|
|
2006-01-06 08:19:28 +00:00
|
|
|
#ifdef CONFIG_COMPAT
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-02-13 13:44:29 +00:00
|
|
|
#define SYSCALL(esame,emu) .long emu
|
2013-04-24 10:58:39 +00:00
|
|
|
.globl sys_call_table_emu
|
2005-04-16 22:20:36 +00:00
|
|
|
sys_call_table_emu:
|
|
|
|
#include "syscalls.S"
|
|
|
|
#undef SYSCALL
|
|
|
|
#endif
|