mirror of
https://github.com/torvalds/linux.git
synced 2024-12-24 11:51:27 +00:00
0aaba41b58
The vdso code for the getcpu() and the clock_gettime() call use the access register mode to access the per-CPU vdso data page with the current code. An alternative to the complicated AR mode is to use the secondary space mode. This makes the vdso faster and quite a bit simpler. The downside is that the uaccess code has to be changed quite a bit. Which instructions are used depends on the machine and what kind of uaccess operation is requested. The instruction dictates which ASCE value needs to be loaded into %cr1 and %cr7. The different cases: * User copy with MVCOS for z10 and newer machines The MVCOS instruction can copy between the primary space (aka user) and the home space (aka kernel) directly. For set_fs(KERNEL_DS) the kernel ASCE is loaded into %cr1. For set_fs(USER_DS) the user space is already loaded in %cr1. * User copy with MVCP/MVCS for older machines To be able to execute the MVCP/MVCS instructions the kernel needs to switch to primary mode. The control register %cr1 has to be set to the kernel ASCE and %cr7 to either the kernel ASCE or the user ASCE dependent on set_fs(KERNEL_DS) vs set_fs(USER_DS). * Data access in the user address space for strnlen / futex To use "normal" instruction with data from the user address space the secondary space mode is used. The kernel needs to switch to primary mode, %cr1 has to contain the kernel ASCE and %cr7 either the user ASCE or the kernel ASCE, dependent on set_fs. To load a new value into %cr1 or %cr7 is an expensive operation, the kernel tries to be lazy about it. E.g. for multiple user copies in a row with MVCP/MVCS the replacement of the vdso ASCE in %cr7 with the user ASCE is done only once. On return to user space a CPU bit is checked that loads the vdso ASCE again. To enable and disable the data access via the secondary space two new functions are added, enable_sacf_uaccess and disable_sacf_uaccess. The fact that a context is in secondary space uaccess mode is stored in the mm_segment_t value for the task. The code of an interrupt may use set_fs as long as it returns to the previous state it got with get_fs with another call to set_fs. The code in finish_arch_post_lock_switch simply has to do a set_fs with the current mm_segment_t value for the task. For CPUs with MVCOS: CPU running in | %cr1 ASCE | %cr7 ASCE | --------------------------------------|-----------|-----------| user space | user | vdso | kernel, USER_DS, normal-mode | user | vdso | kernel, USER_DS, normal-mode, lazy | user | user | kernel, USER_DS, sacf-mode | kernel | user | kernel, KERNEL_DS, normal-mode | kernel | vdso | kernel, KERNEL_DS, normal-mode, lazy | kernel | kernel | kernel, KERNEL_DS, sacf-mode | kernel | kernel | For CPUs without MVCOS: CPU running in | %cr1 ASCE | %cr7 ASCE | --------------------------------------|-----------|-----------| user space | user | vdso | kernel, USER_DS, normal-mode | user | vdso | kernel, USER_DS, normal-mode lazy | kernel | user | kernel, USER_DS, sacf-mode | kernel | user | kernel, KERNEL_DS, normal-mode | kernel | vdso | kernel, KERNEL_DS, normal-mode, lazy | kernel | kernel | kernel, KERNEL_DS, sacf-mode | kernel | kernel | The lines with "lazy" refer to the state after a copy via the secondary space with a delayed reload of %cr1 and %cr7. There are three hardware address spaces that can cause a DAT exception, primary, secondary and home space. The exception can be related to four different fault types: user space fault, vdso fault, kernel fault, and the gmap faults. Dependent on the set_fs state and normal vs. sacf mode there are a number of fault combinations: 1) user address space fault via the primary ASCE 2) gmap address space fault via the primary ASCE 3) kernel address space fault via the primary ASCE for machines with MVCOS and set_fs(KERNEL_DS) 4) vdso address space faults via the secondary ASCE with an invalid address while running in secondary space in problem state 5) user address space fault via the secondary ASCE for user-copy based on the secondary space mode, e.g. futex_ops or strnlen_user 6) kernel address space fault via the secondary ASCE for user-copy with secondary space mode with set_fs(KERNEL_DS) 7) kernel address space fault via the primary ASCE for user-copy with secondary space mode with set_fs(USER_DS) on machines without MVCOS. 8) kernel address space fault via the home space ASCE Replace user_space_fault() with a new function get_fault_type() that can distinguish all four different fault types. With these changes the futex atomic ops from the kernel and the strnlen_user will get a little bit slower, as well as the old style uaccess with MVCP/MVCS. All user accesses based on MVCOS will be as fast as before. On the positive side, the user space vdso code is a lot faster and Linux ceases to use the complicated AR mode. Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com> Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
149 lines
4.0 KiB
ArmAsm
149 lines
4.0 KiB
ArmAsm
/*
|
|
* Userland implementation of clock_gettime() for 64 bits processes in a
|
|
* s390 kernel for use in the vDSO
|
|
*
|
|
* Copyright IBM Corp. 2008
|
|
* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License (version 2 only)
|
|
* as published by the Free Software Foundation.
|
|
*/
|
|
#include <asm/vdso.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/unistd.h>
|
|
|
|
.text
|
|
.align 4
|
|
.globl __kernel_clock_gettime
|
|
.type __kernel_clock_gettime,@function
|
|
__kernel_clock_gettime:
|
|
.cfi_startproc
|
|
aghi %r15,-16
|
|
larl %r5,_vdso_data
|
|
cghi %r2,__CLOCK_REALTIME_COARSE
|
|
je 4f
|
|
cghi %r2,__CLOCK_REALTIME
|
|
je 5f
|
|
cghi %r2,-3 /* Per-thread CPUCLOCK with PID=0, VIRT=1 */
|
|
je 9f
|
|
cghi %r2,__CLOCK_MONOTONIC_COARSE
|
|
je 3f
|
|
cghi %r2,__CLOCK_MONOTONIC
|
|
jne 12f
|
|
|
|
/* CLOCK_MONOTONIC */
|
|
0: lg %r4,__VDSO_UPD_COUNT(%r5) /* load update counter */
|
|
tmll %r4,0x0001 /* pending update ? loop */
|
|
jnz 0b
|
|
stcke 0(%r15) /* Store TOD clock */
|
|
lgf %r2,__VDSO_TK_SHIFT(%r5) /* Timekeeper shift */
|
|
lg %r0,__VDSO_WTOM_SEC(%r5)
|
|
lg %r1,1(%r15)
|
|
sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */
|
|
msgf %r1,__VDSO_TK_MULT(%r5) /* * tk->mult */
|
|
alg %r1,__VDSO_WTOM_NSEC(%r5)
|
|
srlg %r1,%r1,0(%r2) /* >> tk->shift */
|
|
clg %r4,__VDSO_UPD_COUNT(%r5) /* check update counter */
|
|
jne 0b
|
|
larl %r5,13f
|
|
1: clg %r1,0(%r5)
|
|
jl 2f
|
|
slg %r1,0(%r5)
|
|
aghi %r0,1
|
|
j 1b
|
|
2: stg %r0,0(%r3) /* store tp->tv_sec */
|
|
stg %r1,8(%r3) /* store tp->tv_nsec */
|
|
lghi %r2,0
|
|
aghi %r15,16
|
|
br %r14
|
|
|
|
/* CLOCK_MONOTONIC_COARSE */
|
|
3: lg %r4,__VDSO_UPD_COUNT(%r5) /* load update counter */
|
|
tmll %r4,0x0001 /* pending update ? loop */
|
|
jnz 3b
|
|
lg %r0,__VDSO_WTOM_CRS_SEC(%r5)
|
|
lg %r1,__VDSO_WTOM_CRS_NSEC(%r5)
|
|
clg %r4,__VDSO_UPD_COUNT(%r5) /* check update counter */
|
|
jne 3b
|
|
j 2b
|
|
|
|
/* CLOCK_REALTIME_COARSE */
|
|
4: lg %r4,__VDSO_UPD_COUNT(%r5) /* load update counter */
|
|
tmll %r4,0x0001 /* pending update ? loop */
|
|
jnz 4b
|
|
lg %r0,__VDSO_XTIME_CRS_SEC(%r5)
|
|
lg %r1,__VDSO_XTIME_CRS_NSEC(%r5)
|
|
clg %r4,__VDSO_UPD_COUNT(%r5) /* check update counter */
|
|
jne 4b
|
|
j 7f
|
|
|
|
/* CLOCK_REALTIME */
|
|
5: lg %r4,__VDSO_UPD_COUNT(%r5) /* load update counter */
|
|
tmll %r4,0x0001 /* pending update ? loop */
|
|
jnz 5b
|
|
stcke 0(%r15) /* Store TOD clock */
|
|
lg %r1,1(%r15)
|
|
lg %r0,__VDSO_TS_END(%r5) /* TOD steering end time */
|
|
slgr %r0,%r1 /* now - ts_steering_end */
|
|
ltgr %r0,%r0 /* past end of steering ? */
|
|
jm 17f
|
|
srlg %r0,%r0,15 /* 1 per 2^16 */
|
|
tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */
|
|
jz 18f
|
|
lcgr %r0,%r0 /* negative TOD offset */
|
|
18: algr %r1,%r0 /* add steering offset */
|
|
17: lgf %r2,__VDSO_TK_SHIFT(%r5) /* Timekeeper shift */
|
|
sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */
|
|
msgf %r1,__VDSO_TK_MULT(%r5) /* * tk->mult */
|
|
alg %r1,__VDSO_XTIME_NSEC(%r5) /* + tk->xtime_nsec */
|
|
srlg %r1,%r1,0(%r2) /* >> tk->shift */
|
|
lg %r0,__VDSO_XTIME_SEC(%r5) /* tk->xtime_sec */
|
|
clg %r4,__VDSO_UPD_COUNT(%r5) /* check update counter */
|
|
jne 5b
|
|
larl %r5,13f
|
|
6: clg %r1,0(%r5)
|
|
jl 7f
|
|
slg %r1,0(%r5)
|
|
aghi %r0,1
|
|
j 6b
|
|
7: stg %r0,0(%r3) /* store tp->tv_sec */
|
|
stg %r1,8(%r3) /* store tp->tv_nsec */
|
|
lghi %r2,0
|
|
aghi %r15,16
|
|
br %r14
|
|
|
|
/* CPUCLOCK_VIRT for this thread */
|
|
9: lghi %r4,0
|
|
icm %r0,15,__VDSO_ECTG_OK(%r5)
|
|
jz 12f
|
|
sacf 256 /* Magic ectg instruction */
|
|
.insn ssf,0xc80100000000,__VDSO_ECTG_BASE(4),__VDSO_ECTG_USER(4),4
|
|
sacf 0
|
|
algr %r1,%r0 /* r1 = cputime as TOD value */
|
|
mghi %r1,1000 /* convert to nanoseconds */
|
|
srlg %r1,%r1,12 /* r1 = cputime in nanosec */
|
|
lgr %r4,%r1
|
|
larl %r5,13f
|
|
srlg %r1,%r1,9 /* divide by 1000000000 */
|
|
mlg %r0,8(%r5)
|
|
srlg %r0,%r0,11 /* r0 = tv_sec */
|
|
stg %r0,0(%r3)
|
|
msg %r0,0(%r5) /* calculate tv_nsec */
|
|
slgr %r4,%r0 /* r4 = tv_nsec */
|
|
stg %r4,8(%r3)
|
|
lghi %r2,0
|
|
aghi %r15,16
|
|
br %r14
|
|
|
|
/* Fallback to system call */
|
|
12: lghi %r1,__NR_clock_gettime
|
|
svc 0
|
|
aghi %r15,16
|
|
br %r14
|
|
|
|
13: .quad 1000000000
|
|
14: .quad 19342813113834067
|
|
.cfi_endproc
|
|
.size __kernel_clock_gettime,.-__kernel_clock_gettime
|