Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-10-23 05:30:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

s390 updates for 6.11 merge window

Browse Source 
- Remove restrictions on PAI NNPA and crypto counters, enabling
   concurrent per-task and system-wide sampling and counting events
 
 - Switch to GENERIC_CPU_DEVICES by setting up the CPU present mask in
   the architecture code and letting the generic code handle CPU bring-up
 
 - Add support for the diag204 busy indication facility to prevent
   undesirable blocking during hypervisor logical CPU utilization
   queries. Implement results caching
 
 - Improve the handling of Store Data SCLP events by suppressing
   unnecessary warning, preventing buffer release in I/O during failures,
   and adding timeout handling for Store Data requests to address potential
   firmware issues
 
 - Provide optimized __arch_hweight*() implementations
 
 - Remove the unnecessary CPU KOBJ_CHANGE uevents generated during topology
   updates, as they are unused and also not present on other architectures
 
 - Cleanup atomic_ops, optimize __atomic_set() for small values and
   __atomic_cmpxchg_bool() for compilers supporting flag output constraint
 
 - Couple of cleanups for KVM:
   - Move and improve KVM struct definitions for DAT tables from gaccess.c
     to a new header
   - Pass the asce as parameter to sie64a()
 
 - Make the crdte() and cspg() page table handling wrappers return a
   boolean to indicate success, like the other existing "compare and swap"
   wrappers
 
 - Add documentation for HWCAP flags
 
 - Switch to obtaining total RAM pages from memblock instead of
   totalram_pages() during mm init, to ensure correct calculation of zero
   page size, when defer_init is enabled
 
 - Refactor lowcore access and switch to using the get_lowcore() function
   instead of the S390_lowcore macro
 
 - Cleanups for PG_arch_1 and folio handling in UV and hugetlb code
 
 - Add missing MODULE_DESCRIPTION() macros
 
 - Fix VM_FAULT_HWPOISON handling in do_exception()
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEE3QHqV+H2a8xAv27vjYWKoQLXFBgFAmaYGegACgkQjYWKoQLX
 FBjCwwf/aRYoLIXCa9/nHGWFiUjZm6xBgVwZh55bXjfNG9TI2J9UZSsYlOFGUJKl
 gvD2Ym+LqAejK8R4EUHkfD6ftaKMQuIxNDoedxhwuSpfOQ2mZ5teu0MxTh8QcUAx
 4Y2w5XEeCuqE3SuoZ4SJa58K4rGl4cFpPsKNa8ofdzH1ZLFNe8Wqzis4kh0htqLb
 FtPj6nsgfzQ5kg14rVkGxCa4CqoFxonXgsA6nH6xZLbxKUInyq8uV44UBQ+aJq5v
 dsdzZ5XuAJHN2FpBuuOYQYZYw3XIy/kka7o4EjffORi5SGCRMWO4Zt0P6HXaNkh6
 xV8EEO8myeo7rV8dnrk1V4yGjGJmfA==
 =3IGY
 -----END PGP SIGNATURE-----

Merge tag 's390-6.11-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux

Pull s390 updates from Vasily Gorbik:

 - Remove restrictions on PAI NNPA and crypto counters, enabling
   concurrent per-task and system-wide sampling and counting events

 - Switch to GENERIC_CPU_DEVICES by setting up the CPU present mask in
   the architecture code and letting the generic code handle CPU
   bring-up

 - Add support for the diag204 busy indication facility to prevent
   undesirable blocking during hypervisor logical CPU utilization
   queries. Implement results caching

 - Improve the handling of Store Data SCLP events by suppressing
   unnecessary warning, preventing buffer release in I/O during
   failures, and adding timeout handling for Store Data requests to
   address potential firmware issues

 - Provide optimized __arch_hweight*() implementations

 - Remove the unnecessary CPU KOBJ_CHANGE uevents generated during
   topology updates, as they are unused and also not present on other
   architectures

 - Cleanup atomic_ops, optimize __atomic_set() for small values and
   __atomic_cmpxchg_bool() for compilers supporting flag output
   constraint

 - Couple of cleanups for KVM:
     - Move and improve KVM struct definitions for DAT tables from
       gaccess.c to a new header
     - Pass the asce as parameter to sie64a()

 - Make the crdte() and cspg() page table handling wrappers return a
   boolean to indicate success, like the other existing "compare and
   swap" wrappers

 - Add documentation for HWCAP flags

 - Switch to obtaining total RAM pages from memblock instead of
   totalram_pages() during mm init, to ensure correct calculation of
   zero page size, when defer_init is enabled

 - Refactor lowcore access and switch to using the get_lowcore()
   function instead of the S390_lowcore macro

 - Cleanups for PG_arch_1 and folio handling in UV and hugetlb code

 - Add missing MODULE_DESCRIPTION() macros

 - Fix VM_FAULT_HWPOISON handling in do_exception()

* tag 's390-6.11-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (54 commits)
  s390/mm: Fix VM_FAULT_HWPOISON handling in do_exception()
  s390/kvm: Move bitfields for dat tables
  s390/entry: Pass the asce as parameter to sie64a()
  s390/sthyi: Use cached data when diag is busy
  s390/sthyi: Move diag operations
  s390/hypfs_diag: Diag204 busy loop
  s390/diag: Add busy-indication-facility requirements
  s390/diag: Diag204 add busy return errno
  s390/diag: Return errno's from diag204
  s390/sclp: Diag204 busy indication facility detection
  s390/atomic_ops: Make use of flag output constraint
  s390/atomic_ops: Improve __atomic_set() for small values
  s390/atomic_ops: Use symbolic names
  s390/smp: Switch to GENERIC_CPU_DEVICES
  s390/hwcaps: Add documentation for HWCAP flags
  s390/pgtable: Make crdte() and cspg() return a value
  s390/topology: Remove CPU KOBJ_CHANGE uevents
  s390/sclp: Add timeout to Store Data requests
  s390/sclp: Prevent release of buffer in I/O
  s390/sclp: Suppress unnecessary Store Data warning
  ...
This commit is contained in:
Linus Torvalds 2024-07-18 15:41:45 -07:00
commit 1c7d0c3af5
88 changed files with 1207 additions and 778 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
arch/s390/Kconfig
View File

@ -21,7 +21,7 @@ config ARCH_PROC_KCORE_TEXT
def_bool y
config GENERIC_HWEIGHT
def_bool y
def_bool !HAVE_MARCH_Z196_FEATURES
config GENERIC_BUG
def_bool y if BUG
@ -142,6 +142,7 @@ config S390
select FUNCTION_ALIGNMENT_8B if CC_IS_GCC
select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC
select GENERIC_ALLOCATOR
select GENERIC_CPU_DEVICES
select GENERIC_CPU_AUTOPROBE
select GENERIC_CPU_VULNERABILITIES
select GENERIC_ENTRY

4
arch/s390/boot/ipl_parm.c
View File

@ -51,11 +51,11 @@ static inline int __diag308(unsigned long subcode, void *addr)
: [r1] "+&d" (r1.pair),
[reg1] "=&d" (reg1),
[reg2] "=&a" (reg2),
"+Q" (S390_lowcore.program_new_psw),
"+Q" (get_lowcore()->program_new_psw),
"=Q" (old)
: [subcode] "d" (subcode),
[psw_old] "a" (&old),
[psw_pgm] "a" (&S390_lowcore.program_new_psw)
[psw_pgm] "a" (&get_lowcore()->program_new_psw)
: "cc", "memory");
return r1.odd;
}

2
arch/s390/boot/ipl_report.c
View File

@ -106,7 +106,7 @@ int read_ipl_report(void)
* the IPL parameter list, then align the address to a double
* word boundary.
*/
tmp = (unsigned long) S390_lowcore.ipl_parmblock_ptr;
tmp = (unsigned long)get_lowcore()->ipl_parmblock_ptr;
pl_hdr = (struct ipl_pl_hdr *) tmp;
tmp = (tmp + pl_hdr->len + 7) & -8UL;
rl_hdr = (struct ipl_rl_hdr *) tmp;

18
arch/s390/boot/pgm_check_info.c
View File

@ -145,22 +145,22 @@ void print_stacktrace(unsigned long sp)
void print_pgm_check_info(void)
{
unsigned long *gpregs = (unsigned long *)S390_lowcore.gpregs_save_area;
struct psw_bits *psw = &psw_bits(S390_lowcore.psw_save_area);
unsigned long *gpregs = (unsigned long *)get_lowcore()->gpregs_save_area;
struct psw_bits *psw = &psw_bits(get_lowcore()->psw_save_area);
decompressor_printk("Linux version %s\n", kernel_version);
if (!is_prot_virt_guest() && early_command_line[0])
decompressor_printk("Kernel command line: %s\n", early_command_line);
decompressor_printk("Kernel fault: interruption code %04x ilc:%x\n",
S390_lowcore.pgm_code, S390_lowcore.pgm_ilc >> 1);
get_lowcore()->pgm_code, get_lowcore()->pgm_ilc >> 1);
if (kaslr_enabled()) {
decompressor_printk("Kernel random base: %lx\n", __kaslr_offset);
decompressor_printk("Kernel random base phys: %lx\n", __kaslr_offset_phys);
}
decompressor_printk("PSW : %016lx %016lx (%pS)\n",
S390_lowcore.psw_save_area.mask,
S390_lowcore.psw_save_area.addr,
(void *)S390_lowcore.psw_save_area.addr);
get_lowcore()->psw_save_area.mask,
get_lowcore()->psw_save_area.addr,
(void *)get_lowcore()->psw_save_area.addr);
decompressor_printk(
" R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x P:%x AS:%x CC:%x PM:%x RI:%x EA:%x\n",
psw->per, psw->dat, psw->io, psw->ext, psw->key, psw->mcheck,
@ -174,8 +174,8 @@ void print_pgm_check_info(void)
gpregs[8], gpregs[9], gpregs[10], gpregs[11]);
decompressor_printk(" %016lx %016lx %016lx %016lx\n",
gpregs[12], gpregs[13], gpregs[14], gpregs[15]);
print_stacktrace(S390_lowcore.gpregs_save_area[15]);
print_stacktrace(get_lowcore()->gpregs_save_area[15]);
decompressor_printk("Last Breaking-Event-Address:\n");
decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)S390_lowcore.pgm_last_break,
(void *)S390_lowcore.pgm_last_break);
decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)get_lowcore()->pgm_last_break,
(void *)get_lowcore()->pgm_last_break);
}

8
arch/s390/boot/physmem_info.c
View File

@ -81,11 +81,11 @@ static int __diag260(unsigned long rx1, unsigned long rx2)
[reg2] "=&a" (reg2),
[rc] "+&d" (rc),
[ry] "+&d" (ry),
"+Q" (S390_lowcore.program_new_psw),
"+Q" (get_lowcore()->program_new_psw),
"=Q" (old)
: [rx] "d" (rx.pair),
[psw_old] "a" (&old),
[psw_pgm] "a" (&S390_lowcore.program_new_psw)
[psw_pgm] "a" (&get_lowcore()->program_new_psw)
: "cc", "memory");
return rc == 0 ? ry : -1;
}
@ -129,10 +129,10 @@ static int tprot(unsigned long addr)
: [reg1] "=&d" (reg1),
[reg2] "=&a" (reg2),
[rc] "+&d" (rc),
"=Q" (S390_lowcore.program_new_psw.addr),
"=Q" (get_lowcore()->program_new_psw.addr),
"=Q" (old)
: [psw_old] "a" (&old),
[psw_pgm] "a" (&S390_lowcore.program_new_psw),
[psw_pgm] "a" (&get_lowcore()->program_new_psw),
[addr] "a" (addr)
: "cc", "memory");
return rc;

12
arch/s390/boot/startup.c
View File

@ -78,10 +78,10 @@ static int cmma_test_essa(void)
[reg2] "=&a" (reg2),
[rc] "+&d" (rc),
[tmp] "=&d" (tmp),
"+Q" (S390_lowcore.program_new_psw),
"+Q" (get_lowcore()->program_new_psw),
"=Q" (old)
: [psw_old] "a" (&old),
[psw_pgm] "a" (&S390_lowcore.program_new_psw),
[psw_pgm] "a" (&get_lowcore()->program_new_psw),
[cmd] "i" (ESSA_GET_STATE)
: "cc", "memory");
return rc;
@ -101,10 +101,10 @@ static void cmma_init(void)
static void setup_lpp(void)
{
S390_lowcore.current_pid = 0;
S390_lowcore.lpp = LPP_MAGIC;
get_lowcore()->current_pid = 0;
get_lowcore()->lpp = LPP_MAGIC;
if (test_facility(40))
lpp(&S390_lowcore.lpp);
lpp(&get_lowcore()->lpp);
}
#ifdef CONFIG_KERNEL_UNCOMPRESSED
@ -501,7 +501,7 @@ void startup_kernel(void)
* Save KASLR offset for early dumps, before vmcore_info is set.
* Mark as uneven to distinguish from real vmcore_info pointer.
*/
S390_lowcore.vmcore_info = __kaslr_offset_phys ? __kaslr_offset_phys | 0x1UL : 0;
get_lowcore()->vmcore_info = __kaslr_offset_phys ? __kaslr_offset_phys | 0x1UL : 0;
/*
* Jump to the decompressed kernel entry point and switch DAT mode on.

12
arch/s390/boot/vmem.c
View File

@ -476,13 +476,13 @@ void setup_vmem(unsigned long kernel_start, unsigned long kernel_end, unsigned l
kasan_populate_shadow(kernel_start, kernel_end);
S390_lowcore.kernel_asce.val = swapper_pg_dir | asce_bits;
S390_lowcore.user_asce = s390_invalid_asce;
get_lowcore()->kernel_asce.val = swapper_pg_dir | asce_bits;
get_lowcore()->user_asce = s390_invalid_asce;
local_ctl_load(1, &S390_lowcore.kernel_asce);
local_ctl_load(7, &S390_lowcore.user_asce);
local_ctl_load(13, &S390_lowcore.kernel_asce);
local_ctl_load(1, &get_lowcore()->kernel_asce);
local_ctl_load(7, &get_lowcore()->user_asce);
local_ctl_load(13, &get_lowcore()->kernel_asce);
init_mm.context.asce = S390_lowcore.kernel_asce.val;
init_mm.context.asce = get_lowcore()->kernel_asce.val;
init_mm.pgd = init_mm_pgd;
}

1
arch/s390/crypto/crc32-vx.c
View File

@ -297,6 +297,7 @@ module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
module_exit(crc_vx_mod_exit);
MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("crc32");

4
arch/s390/hypfs/hypfs_dbfs.c
View File

@ -39,7 +39,9 @@ static ssize_t dbfs_read(struct file *file, char __user *buf,
return 0;
df = file_inode(file)->i_private;
mutex_lock(&df->lock);
if (mutex_lock_interruptible(&df->lock))
return -ERESTARTSYS;
data = hypfs_dbfs_data_alloc(df);
if (!data) {
mutex_unlock(&df->lock);

17
arch/s390/hypfs/hypfs_diag.c
View File

@ -140,11 +140,22 @@ fail_alloc:
int diag204_store(void *buf, int pages)
{
unsigned long subcode;
int rc;
rc = diag204((unsigned long)diag204_store_sc |
(unsigned long)diag204_get_info_type(), pages, buf);
return rc < 0 ? -EOPNOTSUPP : 0;
subcode = diag204_get_info_type();
subcode |= diag204_store_sc;
if (diag204_has_bif())
subcode |= DIAG204_BIF_BIT;
while (1) {
rc = diag204(subcode, pages, buf);
if (rc != -EBUSY)
break;
if (signal_pending(current))
return -ERESTARTSYS;
schedule_timeout_interruptible(DIAG204_BUSY_WAIT);
}
return rc < 0 ? rc : 0;
}
struct dbfs_d204_hdr {

76
arch/s390/include/asm/arch_hweight.h Normal file
View File

@ -0,0 +1,76 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_S390_ARCH_HWEIGHT_H
#define _ASM_S390_ARCH_HWEIGHT_H
#include <linux/types.h>
static __always_inline unsigned long popcnt_z196(unsigned long w)
{
unsigned long cnt;
asm volatile(".insn rrf,0xb9e10000,%[cnt],%[w],0,0"
: [cnt] "=d" (cnt)
: [w] "d" (w)
: "cc");
return cnt;
}
static __always_inline unsigned long popcnt_z15(unsigned long w)
{
unsigned long cnt;
asm volatile(".insn rrf,0xb9e10000,%[cnt],%[w],8,0"
: [cnt] "=d" (cnt)
: [w] "d" (w)
: "cc");
return cnt;
}
static __always_inline unsigned long __arch_hweight64(__u64 w)
{
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES))
return popcnt_z15(w);
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) {
w = popcnt_z196(w);
w += w >> 32;
w += w >> 16;
w += w >> 8;
return w & 0xff;
}
return __sw_hweight64(w);
}
static __always_inline unsigned int __arch_hweight32(unsigned int w)
{
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES))
return popcnt_z15(w);
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) {
w = popcnt_z196(w);
w += w >> 16;
w += w >> 8;
return w & 0xff;
}
return __sw_hweight32(w);
}
static __always_inline unsigned int __arch_hweight16(unsigned int w)
{
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES))
return popcnt_z15((unsigned short)w);
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) {
w = popcnt_z196(w);
w += w >> 8;
return w & 0xff;
}
return __sw_hweight16(w);
}
static __always_inline unsigned int __arch_hweight8(unsigned int w)
{
if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES))
return popcnt_z196((unsigned char)w);
return __sw_hweight8(w);
}
#endif /* _ASM_S390_ARCH_HWEIGHT_H */

84
arch/s390/include/asm/atomic_ops.h
View File

@ -8,21 +8,29 @@
#ifndef __ARCH_S390_ATOMIC_OPS__
#define __ARCH_S390_ATOMIC_OPS__
#include <linux/limits.h>
static __always_inline int __atomic_read(const atomic_t *v)
{
int c;
asm volatile(
" l %0,%1\n"
: "=d" (c) : "R" (v->counter));
" l %[c],%[counter]\n"
: [c] "=d" (c) : [counter] "R" (v->counter));
return c;
}
static __always_inline void __atomic_set(atomic_t *v, int i)
{
asm volatile(
" st %1,%0\n"
: "=R" (v->counter) : "d" (i));
if (__builtin_constant_p(i) && i >= S16_MIN && i <= S16_MAX) {
asm volatile(
" mvhi %[counter], %[i]\n"
: [counter] "=Q" (v->counter) : [i] "K" (i));
} else {
asm volatile(
" st %[i],%[counter]\n"
: [counter] "=R" (v->counter) : [i] "d" (i));
}
}
static __always_inline s64 __atomic64_read(const atomic64_t *v)
@ -30,16 +38,22 @@ static __always_inline s64 __atomic64_read(const atomic64_t *v)
s64 c;
asm volatile(
" lg %0,%1\n"
: "=d" (c) : "RT" (v->counter));
" lg %[c],%[counter]\n"
: [c] "=d" (c) : [counter] "RT" (v->counter));
return c;
}
static __always_inline void __atomic64_set(atomic64_t *v, s64 i)
{
asm volatile(
" stg %1,%0\n"
: "=RT" (v->counter) : "d" (i));
if (__builtin_constant_p(i) && i >= S16_MIN && i <= S16_MAX) {
asm volatile(
" mvghi %[counter], %[i]\n"
: [counter] "=Q" (v->counter) : [i] "K" (i));
} else {
asm volatile(
" stg %[i],%[counter]\n"
: [counter] "=RT" (v->counter) : [i] "d" (i));
}
}
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
@ -164,6 +178,44 @@ static __always_inline int __atomic_cmpxchg(int *ptr, int old, int new)
return old;
}
static __always_inline long __atomic64_cmpxchg(long *ptr, long old, long new)
{
asm volatile(
" csg %[old],%[new],%[ptr]"
: [old] "+d" (old), [ptr] "+QS" (*ptr)
: [new] "d" (new)
: "cc", "memory");
return old;
}
#ifdef __GCC_ASM_FLAG_OUTPUTS__
static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
{
int cc;
asm volatile(
" cs %[old],%[new],%[ptr]"
: [old] "+d" (old), [ptr] "+Q" (*ptr), "=@cc" (cc)
: [new] "d" (new)
: "memory");
return cc == 0;
}
static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)
{
int cc;
asm volatile(
" csg %[old],%[new],%[ptr]"
: [old] "+d" (old), [ptr] "+QS" (*ptr), "=@cc" (cc)
: [new] "d" (new)
: "memory");
return cc == 0;
}
#else /* __GCC_ASM_FLAG_OUTPUTS__ */
static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
{
int old_expected = old;
@ -176,16 +228,6 @@ static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
return old == old_expected;
}
static __always_inline long __atomic64_cmpxchg(long *ptr, long old, long new)
{
asm volatile(
" csg %[old],%[new],%[ptr]"
: [old] "+d" (old), [ptr] "+QS" (*ptr)
: [new] "d" (new)
: "cc", "memory");
return old;
}
static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)
{
long old_expected = old;
@ -198,4 +240,6 @@ static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long ne
return old == old_expected;
}
#endif /* __GCC_ASM_FLAG_OUTPUTS__ */
#endif /* __ARCH_S390_ATOMIC_OPS__ */

3
arch/s390/include/asm/bitops.h
View File

@ -379,8 +379,9 @@ static inline int fls(unsigned int word)
return fls64(word);
}
#include <asm/arch_hweight.h>
#include <asm-generic/bitops/const_hweight.h>
#include <asm-generic/bitops/ffz.h>
#include <asm-generic/bitops/hweight.h>
#include <asm-generic/bitops/sched.h>
#include <asm-generic/bitops/le.h>
#include <asm-generic/bitops/ext2-atomic-setbit.h>

2
arch/s390/include/asm/current.h
View File

@ -14,6 +14,6 @@
struct task_struct;
#define current ((struct task_struct *const)S390_lowcore.current_task)
#define current ((struct task_struct *const)get_lowcore()->current_task)
#endif /* !(_S390_CURRENT_H) */

170
arch/s390/include/asm/dat-bits.h Normal file
View File

@ -0,0 +1,170 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* DAT table and related structures
*
* Copyright IBM Corp. 2024
*
*/
#ifndef _S390_DAT_BITS_H
#define _S390_DAT_BITS_H
union asce {
unsigned long val;
struct {
unsigned long rsto: 52;/* Region- or Segment-Table Origin */
unsigned long : 2;
unsigned long g : 1; /* Subspace Group control */
unsigned long p : 1; /* Private Space control */
unsigned long s : 1; /* Storage-Alteration-Event control */
unsigned long x : 1; /* Space-Switch-Event control */
unsigned long r : 1; /* Real-Space control */
unsigned long : 1;
unsigned long dt : 2; /* Designation-Type control */
unsigned long tl : 2; /* Region- or Segment-Table Length */
};
};
enum {
ASCE_TYPE_SEGMENT = 0,
ASCE_TYPE_REGION3 = 1,
ASCE_TYPE_REGION2 = 2,
ASCE_TYPE_REGION1 = 3
};
union region1_table_entry {
unsigned long val;
struct {
unsigned long rto: 52;/* Region-Table Origin */
unsigned long : 2;
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 1;
unsigned long tf : 2; /* Region-Second-Table Offset */
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long : 1;
unsigned long tt : 2; /* Table-Type Bits */
unsigned long tl : 2; /* Region-Second-Table Length */
};
};
union region2_table_entry {
unsigned long val;
struct {
unsigned long rto: 52;/* Region-Table Origin */
unsigned long : 2;
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 1;
unsigned long tf : 2; /* Region-Third-Table Offset */
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long : 1;
unsigned long tt : 2; /* Table-Type Bits */
unsigned long tl : 2; /* Region-Third-Table Length */
};
};
struct region3_table_entry_fc0 {
unsigned long sto: 52;/* Segment-Table Origin */
unsigned long : 1;
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 1;
unsigned long tf : 2; /* Segment-Table Offset */
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long cr : 1; /* Common-Region Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long tl : 2; /* Segment-Table Length */
};
struct region3_table_entry_fc1 {
unsigned long rfaa: 33;/* Region-Frame Absolute Address */
unsigned long : 14;
unsigned long av : 1; /* ACCF-Validity Control */
unsigned long acc : 4; /* Access-Control Bits */
unsigned long f : 1; /* Fetch-Protection Bit */
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long iep : 1; /* Instruction-Execution-Protection */
unsigned long : 2;
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long cr : 1; /* Common-Region Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
union region3_table_entry {
unsigned long val;
struct region3_table_entry_fc0 fc0;
struct region3_table_entry_fc1 fc1;
struct {
unsigned long : 53;
unsigned long fc: 1; /* Format-Control */
unsigned long : 4;
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long cr: 1; /* Common-Region Bit */
unsigned long tt: 2; /* Table-Type Bits */
unsigned long : 2;
};
};
struct segment_table_entry_fc0 {
unsigned long pto: 53;/* Page-Table Origin */
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 3;
unsigned long i : 1; /* Segment-Invalid Bit */
unsigned long cs : 1; /* Common-Segment Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
struct segment_table_entry_fc1 {
unsigned long sfaa: 44;/* Segment-Frame Absolute Address */
unsigned long : 3;
unsigned long av : 1; /* ACCF-Validity Control */
unsigned long acc : 4; /* Access-Control Bits */
unsigned long f : 1; /* Fetch-Protection Bit */
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long iep : 1; /* Instruction-Execution-Protection */
unsigned long : 2;
unsigned long i : 1; /* Segment-Invalid Bit */
unsigned long cs : 1; /* Common-Segment Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
union segment_table_entry {
unsigned long val;
struct segment_table_entry_fc0 fc0;
struct segment_table_entry_fc1 fc1;
struct {
unsigned long : 53;
unsigned long fc: 1; /* Format-Control */
unsigned long : 4;
unsigned long i : 1; /* Segment-Invalid Bit */
unsigned long cs: 1; /* Common-Segment Bit */
unsigned long tt: 2; /* Table-Type Bits */
unsigned long : 2;
};
};
union page_table_entry {
unsigned long val;
struct {
unsigned long pfra: 52;/* Page-Frame Real Address */
unsigned long z : 1; /* Zero Bit */
unsigned long i : 1; /* Page-Invalid Bit */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long iep : 1; /* Instruction-Execution-Protection */
unsigned long : 8;
};
};
enum {
TABLE_TYPE_SEGMENT = 0,
TABLE_TYPE_REGION3 = 1,
TABLE_TYPE_REGION2 = 2,
TABLE_TYPE_REGION1 = 3
};
#endif /* _S390_DAT_BITS_H */

8
arch/s390/include/asm/diag.h
View File

@ -12,6 +12,7 @@
#include <linux/if_ether.h>
#include <linux/percpu.h>
#include <asm/asm-extable.h>
#include <asm/sclp.h>
#include <asm/cio.h>
enum diag_stat_enum {
@ -117,6 +118,8 @@ enum diag204_sc {
};
#define DIAG204_SUBCODE_MASK 0xffff
#define DIAG204_BIF_BIT 0x80000000
#define DIAG204_BUSY_WAIT (HZ / 10)
/* The two available diag 204 data formats */
enum diag204_format {
@ -326,6 +329,11 @@ union diag318_info {
};
};
static inline bool diag204_has_bif(void)
{
return sclp.has_diag204_bif;
}
int diag204(unsigned long subcode, unsigned long size, void *addr);
int diag224(void *ptr);
int diag26c(void *req, void *resp, enum diag26c_sc subcode);

8
arch/s390/include/asm/elf.h
View File

@ -91,6 +91,14 @@
/* Keep this the last entry. */
#define R_390_NUM 61
/*
* HWCAP flags - for AT_HWCAP
*
* Bits 32-63 are reserved for use by libc.
* Bit 31 is reserved and will be used by libc to determine if a second
* argument is passed to IFUNC resolvers. This will be implemented when
* there is a need for AT_HWCAP2.
*/
enum {
HWCAP_NR_ESAN3 = 0,
HWCAP_NR_ZARCH = 1,

4
arch/s390/include/asm/facility.h
View File

@ -92,8 +92,8 @@ static inline void __stfle(u64 *stfle_fac_list, int size)
asm volatile(
" stfl 0(0)\n"
: "=m" (S390_lowcore.stfl_fac_list));
stfl_fac_list = S390_lowcore.stfl_fac_list;
: "=m" (get_lowcore()->stfl_fac_list));
stfl_fac_list = get_lowcore()->stfl_fac_list;
memcpy(stfle_fac_list, &stfl_fac_list, 4);
nr = 4; /* bytes stored by stfl */
if (stfl_fac_list & 0x01000000) {

6
arch/s390/include/asm/hardirq.h
View File

@ -13,9 +13,9 @@
#include <asm/lowcore.h>
#define local_softirq_pending() (S390_lowcore.softirq_pending)
#define set_softirq_pending(x) (S390_lowcore.softirq_pending = (x))
#define or_softirq_pending(x) (S390_lowcore.softirq_pending |= (x))
#define local_softirq_pending() (get_lowcore()->softirq_pending)
#define set_softirq_pending(x) (get_lowcore()->softirq_pending = (x))
#define or_softirq_pending(x) (get_lowcore()->softirq_pending |= (x))
#define __ARCH_IRQ_STAT
#define __ARCH_IRQ_EXIT_IRQS_DISABLED

7
arch/s390/include/asm/kvm_host.h
View File

@ -1030,11 +1030,12 @@ void kvm_arch_crypto_clear_masks(struct kvm *kvm);
void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
unsigned long *aqm, unsigned long *adm);
int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa);
int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa,
unsigned long gasce);
static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa)
static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce)
{
return __sie64a(virt_to_phys(sie_block), sie_block, rsa);
return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce);
}
extern char sie_exit;

5
arch/s390/include/asm/lowcore.h
View File

@ -213,7 +213,10 @@ struct lowcore {
__u8 pad_0x1900[0x2000-0x1900]; /* 0x1900 */
} __packed __aligned(8192);
#define S390_lowcore (*((struct lowcore *) 0))
static __always_inline struct lowcore *get_lowcore(void)
{
return NULL;
}
extern struct lowcore *lowcore_ptr[];

8
arch/s390/include/asm/mmu_context.h
View File

@ -76,9 +76,9 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *
int cpu = smp_processor_id();
if (next == &init_mm)
S390_lowcore.user_asce = s390_invalid_asce;
get_lowcore()->user_asce = s390_invalid_asce;
else
S390_lowcore.user_asce.val = next->context.asce;
get_lowcore()->user_asce.val = next->context.asce;
cpumask_set_cpu(cpu, &next->context.cpu_attach_mask);
/* Clear previous user-ASCE from CR7 */
local_ctl_load(7, &s390_invalid_asce);
@ -111,7 +111,7 @@ static inline void finish_arch_post_lock_switch(void)
__tlb_flush_mm_lazy(mm);
preempt_enable();
}
local_ctl_load(7, &S390_lowcore.user_asce);
local_ctl_load(7, &get_lowcore()->user_asce);
}
#define activate_mm activate_mm
@ -120,7 +120,7 @@ static inline void activate_mm(struct mm_struct *prev,
{
switch_mm(prev, next, current);
cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
local_ctl_load(7, &S390_lowcore.user_asce);
local_ctl_load(7, &get_lowcore()->user_asce);
}
#include <asm-generic/mmu_context.h>

5
arch/s390/include/asm/page.h
View File

@ -162,6 +162,7 @@ static inline int page_reset_referenced(unsigned long addr)
#define _PAGE_ACC_BITS 0xf0 /* HW access control bits */
struct page;
struct folio;
void arch_free_page(struct page *page, int order);
void arch_alloc_page(struct page *page, int order);
@ -174,6 +175,8 @@ static inline int devmem_is_allowed(unsigned long pfn)
#define HAVE_ARCH_ALLOC_PAGE
#if IS_ENABLED(CONFIG_PGSTE)
int arch_make_folio_accessible(struct folio *folio);
#define HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE
int arch_make_page_accessible(struct page *page);
#define HAVE_ARCH_MAKE_PAGE_ACCESSIBLE
#endif
@ -247,7 +250,9 @@ static inline unsigned long __phys_addr(unsigned long x, bool is_31bit)
#define pfn_to_phys(pfn) ((pfn) << PAGE_SHIFT)
#define phys_to_page(phys) pfn_to_page(phys_to_pfn(phys))
#define phys_to_folio(phys) page_folio(phys_to_page(phys))
#define page_to_phys(page) pfn_to_phys(page_to_pfn(page))
#define folio_to_phys(page) pfn_to_phys(folio_pfn(folio))
static inline void *pfn_to_virt(unsigned long pfn)
{

17
arch/s390/include/asm/pai.h
View File

@ -55,11 +55,11 @@ static __always_inline void pai_kernel_enter(struct pt_regs *regs)
return;
if (!static_branch_unlikely(&pai_key))
return;
if (!S390_lowcore.ccd)
if (!get_lowcore()->ccd)
return;
if (!user_mode(regs))
return;
WRITE_ONCE(S390_lowcore.ccd, S390_lowcore.ccd | PAI_CRYPTO_KERNEL_OFFSET);
WRITE_ONCE(get_lowcore()->ccd, get_lowcore()->ccd | PAI_CRYPTO_KERNEL_OFFSET);
}
static __always_inline void pai_kernel_exit(struct pt_regs *regs)
@ -68,18 +68,15 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs)
return;
if (!static_branch_unlikely(&pai_key))
return;
if (!S390_lowcore.ccd)
if (!get_lowcore()->ccd)
return;
if (!user_mode(regs))
return;
WRITE_ONCE(S390_lowcore.ccd, S390_lowcore.ccd & ~PAI_CRYPTO_KERNEL_OFFSET);
WRITE_ONCE(get_lowcore()->ccd, get_lowcore()->ccd & ~PAI_CRYPTO_KERNEL_OFFSET);
}
enum paievt_mode {
PAI_MODE_NONE,
PAI_MODE_SAMPLING,
PAI_MODE_COUNTING,
};
#define PAI_SAVE_AREA(x) ((x)->hw.event_base)
#define PAI_CPU_MASK(x) ((x)->hw.addr_filters)
#define PAI_SWLIST(x) (&(x)->hw.tp_list)
#endif

2
arch/s390/include/asm/percpu.h
View File

@ -9,7 +9,7 @@
* s390 uses its own implementation for per cpu data, the offset of
* the cpu local data area is cached in the cpu's lowcore memory.
*/
#define __my_cpu_offset S390_lowcore.percpu_offset
#define __my_cpu_offset get_lowcore()->percpu_offset
/*
* For 64 bit module code, the module may be more than 4G above the

33
arch/s390/include/asm/pgtable.h
View File

@ -609,7 +609,15 @@ static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new)
: "cc");
}
static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new)
/**
* cspg() - Compare and Swap and Purge (CSPG)
* @ptr: Pointer to the value to be exchanged
* @old: The expected old value
* @new: The new value
*
* Return: True if compare and swap was successful, otherwise false.
*/
static inline bool cspg(unsigned long *ptr, unsigned long old, unsigned long new)
{
union register_pair r1 = { .even = old, .odd = new, };
unsigned long address = (unsigned long)ptr | 1;
@ -619,6 +627,7 @@ static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new
: [r1] "+&d" (r1.pair), "+m" (*ptr)
: [address] "d" (address)
: "cc");
return old == r1.even;
}
#define CRDTE_DTT_PAGE 0x00UL
@ -627,7 +636,18 @@ static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new
#define CRDTE_DTT_REGION2 0x18UL
#define CRDTE_DTT_REGION1 0x1cUL
static inline void crdte(unsigned long old, unsigned long new,
/**
* crdte() - Compare and Replace DAT Table Entry
* @old: The expected old value
* @new: The new value
* @table: Pointer to the value to be exchanged
* @dtt: Table type of the table to be exchanged
* @address: The address mapped by the entry to be replaced
* @asce: The ASCE of this entry
*
* Return: True if compare and replace was successful, otherwise false.
*/
static inline bool crdte(unsigned long old, unsigned long new,
unsigned long *table, unsigned long dtt,
unsigned long address, unsigned long asce)
{
@ -638,6 +658,7 @@ static inline void crdte(unsigned long old, unsigned long new,
: [r1] "+&d" (r1.pair)
: [r2] "d" (r2.pair), [asce] "a" (asce)
: "memory", "cc");
return old == r1.even;
}
/*
@ -1167,7 +1188,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
/* At this point the reference through the mapping is still present */
if (mm_is_protected(mm) && pte_present(res))
uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK);
uv_convert_from_secure_pte(res);
return res;
}
@ -1185,7 +1206,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
/* At this point the reference through the mapping is still present */
if (mm_is_protected(vma->vm_mm) && pte_present(res))
uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK);
uv_convert_from_secure_pte(res);
return res;
}
@ -1217,14 +1238,14 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
* The notifier should have destroyed all protected vCPUs at this
* point, so the destroy should be successful.
*/
if (full && !uv_destroy_owned_page(pte_val(res) & PAGE_MASK))
if (full && !uv_destroy_pte(res))
return res;
/*
* If something went wrong and the page could not be destroyed, or
* if this is not a mm teardown, the slower export is used as
* fallback instead.
*/
uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK);
uv_convert_from_secure_pte(res);
return res;
}

30
arch/s390/include/asm/preempt.h
View File

@ -14,7 +14,7 @@
static __always_inline int preempt_count(void)
{
return READ_ONCE(S390_lowcore.preempt_count) & ~PREEMPT_NEED_RESCHED;
return READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED;
}
static __always_inline void preempt_count_set(int pc)
@ -22,26 +22,26 @@ static __always_inline void preempt_count_set(int pc)
int old, new;
do {
old = READ_ONCE(S390_lowcore.preempt_count);
old = READ_ONCE(get_lowcore()->preempt_count);
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
} while (__atomic_cmpxchg(&S390_lowcore.preempt_count,
} while (__atomic_cmpxchg(&get_lowcore()->preempt_count,
old, new) != old);
}
static __always_inline void set_preempt_need_resched(void)
{
__atomic_and(~PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count);
__atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
}
static __always_inline void clear_preempt_need_resched(void)
{
__atomic_or(PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count);
__atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
}
static __always_inline bool test_preempt_need_resched(void)
{
return !(READ_ONCE(S390_lowcore.preempt_count) & PREEMPT_NEED_RESCHED);
return !(READ_ONCE(get_lowcore()->preempt_count) & PREEMPT_NEED_RESCHED);
}
static __always_inline void __preempt_count_add(int val)
@ -52,11 +52,11 @@ static __always_inline void __preempt_count_add(int val)
*/
if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES)) {
if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) {
__atomic_add_const(val, &S390_lowcore.preempt_count);
__atomic_add_const(val, &get_lowcore()->preempt_count);
return;
}
}
__atomic_add(val, &S390_lowcore.preempt_count);
__atomic_add(val, &get_lowcore()->preempt_count);
}
static __always_inline void __preempt_count_sub(int val)
@ -66,12 +66,12 @@ static __always_inline void __preempt_count_sub(int val)
static __always_inline bool __preempt_count_dec_and_test(void)
{
return __atomic_add(-1, &S390_lowcore.preempt_count) == 1;
return __atomic_add(-1, &get_lowcore()->preempt_count) == 1;
}
static __always_inline bool should_resched(int preempt_offset)
{
return unlikely(READ_ONCE(S390_lowcore.preempt_count) ==
return unlikely(READ_ONCE(get_lowcore()->preempt_count) ==
preempt_offset);
}
@ -81,12 +81,12 @@ static __always_inline bool should_resched(int preempt_offset)
static __always_inline int preempt_count(void)
{
return READ_ONCE(S390_lowcore.preempt_count);
return READ_ONCE(get_lowcore()->preempt_count);
}
static __always_inline void preempt_count_set(int pc)
{
S390_lowcore.preempt_count = pc;
get_lowcore()->preempt_count = pc;
}
static __always_inline void set_preempt_need_resched(void)
@ -104,17 +104,17 @@ static __always_inline bool test_preempt_need_resched(void)
static __always_inline void __preempt_count_add(int val)
{
S390_lowcore.preempt_count += val;
get_lowcore()->preempt_count += val;
}
static __always_inline void __preempt_count_sub(int val)
{
S390_lowcore.preempt_count -= val;
get_lowcore()->preempt_count -= val;
}
static __always_inline bool __preempt_count_dec_and_test(void)
{
return !--S390_lowcore.preempt_count && tif_need_resched();
return !--get_lowcore()->preempt_count && tif_need_resched();
}
static __always_inline bool should_resched(int preempt_offset)

8
arch/s390/include/asm/processor.h
View File

@ -46,17 +46,17 @@ typedef long (*sys_call_ptr_t)(struct pt_regs *regs);
static __always_inline void set_cpu_flag(int flag)
{
S390_lowcore.cpu_flags |= (1UL << flag);
get_lowcore()->cpu_flags |= (1UL << flag);
}
static __always_inline void clear_cpu_flag(int flag)
{
S390_lowcore.cpu_flags &= ~(1UL << flag);
get_lowcore()->cpu_flags &= ~(1UL << flag);
}
static __always_inline bool test_cpu_flag(int flag)
{
return S390_lowcore.cpu_flags & (1UL << flag);
return get_lowcore()->cpu_flags & (1UL << flag);
}
static __always_inline bool test_and_set_cpu_flag(int flag)
@ -269,7 +269,7 @@ static __always_inline unsigned long __current_stack_pointer(void)
static __always_inline bool on_thread_stack(void)
{
unsigned long ksp = S390_lowcore.kernel_stack;
unsigned long ksp = get_lowcore()->kernel_stack;
return !((ksp ^ current_stack_pointer) & ~(THREAD_SIZE - 1));
}

1
arch/s390/include/asm/sclp.h
View File

@ -84,6 +84,7 @@ struct sclp_info {
unsigned char has_ibs : 1;
unsigned char has_skey : 1;
unsigned char has_kss : 1;
unsigned char has_diag204_bif : 1;
unsigned char has_gisaf : 1;
unsigned char has_diag318 : 1;
unsigned char has_diag320 : 1;

34
arch/s390/include/asm/setup.h
View File

@ -77,24 +77,24 @@ extern unsigned long max_mappable;
/* The Write Back bit position in the physaddr is given by the SLPC PCI */
extern unsigned long mio_wb_bit_mask;
#define MACHINE_IS_VM (S390_lowcore.machine_flags & MACHINE_FLAG_VM)
#define MACHINE_IS_KVM (S390_lowcore.machine_flags & MACHINE_FLAG_KVM)
#define MACHINE_IS_LPAR (S390_lowcore.machine_flags & MACHINE_FLAG_LPAR)
#define MACHINE_IS_VM (get_lowcore()->machine_flags & MACHINE_FLAG_VM)
#define MACHINE_IS_KVM (get_lowcore()->machine_flags & MACHINE_FLAG_KVM)
#define MACHINE_IS_LPAR (get_lowcore()->machine_flags & MACHINE_FLAG_LPAR)
#define MACHINE_HAS_DIAG9C (S390_lowcore.machine_flags & MACHINE_FLAG_DIAG9C)
#define MACHINE_HAS_ESOP (S390_lowcore.machine_flags & MACHINE_FLAG_ESOP)
#define MACHINE_HAS_IDTE (S390_lowcore.machine_flags & MACHINE_FLAG_IDTE)
#define MACHINE_HAS_EDAT1 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT1)
#define MACHINE_HAS_EDAT2 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT2)
#define MACHINE_HAS_TOPOLOGY (S390_lowcore.machine_flags & MACHINE_FLAG_TOPOLOGY)
#define MACHINE_HAS_TE (S390_lowcore.machine_flags & MACHINE_FLAG_TE)
#define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC)
#define MACHINE_HAS_TLB_GUEST (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_GUEST)
#define MACHINE_HAS_NX (S390_lowcore.machine_flags & MACHINE_FLAG_NX)
#define MACHINE_HAS_GS (S390_lowcore.machine_flags & MACHINE_FLAG_GS)
#define MACHINE_HAS_SCC (S390_lowcore.machine_flags & MACHINE_FLAG_SCC)
#define MACHINE_HAS_PCI_MIO (S390_lowcore.machine_flags & MACHINE_FLAG_PCI_MIO)
#define MACHINE_HAS_RDP (S390_lowcore.machine_flags & MACHINE_FLAG_RDP)
#define MACHINE_HAS_DIAG9C (get_lowcore()->machine_flags & MACHINE_FLAG_DIAG9C)
#define MACHINE_HAS_ESOP (get_lowcore()->machine_flags & MACHINE_FLAG_ESOP)
#define MACHINE_HAS_IDTE (get_lowcore()->machine_flags & MACHINE_FLAG_IDTE)
#define MACHINE_HAS_EDAT1 (get_lowcore()->machine_flags & MACHINE_FLAG_EDAT1)
#define MACHINE_HAS_EDAT2 (get_lowcore()->machine_flags & MACHINE_FLAG_EDAT2)
#define MACHINE_HAS_TOPOLOGY (get_lowcore()->machine_flags & MACHINE_FLAG_TOPOLOGY)
#define MACHINE_HAS_TE (get_lowcore()->machine_flags & MACHINE_FLAG_TE)
#define MACHINE_HAS_TLB_LC (get_lowcore()->machine_flags & MACHINE_FLAG_TLB_LC)
#define MACHINE_HAS_TLB_GUEST (get_lowcore()->machine_flags & MACHINE_FLAG_TLB_GUEST)
#define MACHINE_HAS_NX (get_lowcore()->machine_flags & MACHINE_FLAG_NX)
#define MACHINE_HAS_GS (get_lowcore()->machine_flags & MACHINE_FLAG_GS)
#define MACHINE_HAS_SCC (get_lowcore()->machine_flags & MACHINE_FLAG_SCC)
#define MACHINE_HAS_PCI_MIO (get_lowcore()->machine_flags & MACHINE_FLAG_PCI_MIO)
#define MACHINE_HAS_RDP (get_lowcore()->machine_flags & MACHINE_FLAG_RDP)
/*
* Console mode. Override with conmode=

4
arch/s390/include/asm/smp.h
View File

@ -11,7 +11,7 @@
#include <asm/lowcore.h>
#include <asm/processor.h>
#define raw_smp_processor_id() (S390_lowcore.cpu_nr)
#define raw_smp_processor_id() (get_lowcore()->cpu_nr)
extern struct mutex smp_cpu_state_mutex;
extern unsigned int smp_cpu_mt_shift;
@ -59,7 +59,7 @@ static inline void smp_cpus_done(unsigned int max_cpus)
{
}
extern int smp_rescan_cpus(void);
extern int smp_rescan_cpus(bool early);
extern void __noreturn cpu_die(void);
extern void __cpu_die(unsigned int cpu);
extern int __cpu_disable(void);

2
arch/s390/include/asm/softirq_stack.h
View File

@ -8,7 +8,7 @@
#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
static inline void do_softirq_own_stack(void)
{
call_on_stack(0, S390_lowcore.async_stack, void, __do_softirq);
call_on_stack(0, get_lowcore()->async_stack, void, __do_softirq);
}
#endif
#endif /* __ASM_S390_SOFTIRQ_STACK_H */

2
arch/s390/include/asm/spinlock.h
View File

@ -16,7 +16,7 @@
#include <asm/processor.h>
#include <asm/alternative.h>
#define SPINLOCK_LOCKVAL (S390_lowcore.spinlock_lockval)
#define SPINLOCK_LOCKVAL (get_lowcore()->spinlock_lockval)
extern int spin_retry;

1
arch/s390/include/asm/stacktrace.h
View File

@ -65,6 +65,7 @@ struct stack_frame {
unsigned long sie_reason;
unsigned long sie_flags;
unsigned long sie_control_block_phys;
unsigned long sie_guest_asce;
};
};
unsigned long gprs[10];

10
arch/s390/include/asm/timex.h
View File

@ -161,16 +161,16 @@ static inline unsigned long local_tick_disable(void)
{
unsigned long old;
old = S390_lowcore.clock_comparator;
S390_lowcore.clock_comparator = clock_comparator_max;
set_clock_comparator(S390_lowcore.clock_comparator);
old = get_lowcore()->clock_comparator;
get_lowcore()->clock_comparator = clock_comparator_max;
set_clock_comparator(get_lowcore()->clock_comparator);
return old;
}
static inline void local_tick_enable(unsigned long comp)
{
S390_lowcore.clock_comparator = comp;
set_clock_comparator(S390_lowcore.clock_comparator);
get_lowcore()->clock_comparator = comp;
set_clock_comparator(get_lowcore()->clock_comparator);
}
#define CLOCK_TICK_RATE 1193180 /* Underlying HZ */

12
arch/s390/include/asm/uv.h
View File

@ -483,9 +483,9 @@ static inline int is_prot_virt_host(void)
int uv_pin_shared(unsigned long paddr);
int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb);
int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr);
int uv_destroy_owned_page(unsigned long paddr);
int uv_convert_from_secure(unsigned long paddr);
int uv_convert_owned_from_secure(unsigned long paddr);
int uv_destroy_folio(struct folio *folio);
int uv_destroy_pte(pte_t pte);
int uv_convert_from_secure_pte(pte_t pte);
int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr);
void setup_uv(void);
@ -498,17 +498,17 @@ static inline int uv_pin_shared(unsigned long paddr)
return 0;
}
static inline int uv_destroy_owned_page(unsigned long paddr)
static inline int uv_destroy_folio(struct folio *folio)
{
return 0;
}
static inline int uv_convert_from_secure(unsigned long paddr)
static inline int uv_destroy_pte(pte_t pte)
{
return 0;
}
static inline int uv_convert_owned_from_secure(unsigned long paddr)
static inline int uv_convert_from_secure_pte(pte_t pte)
{
return 0;
}

16
arch/s390/include/asm/vtime.h
View File

@ -4,16 +4,20 @@
static inline void update_timer_sys(void)
{
S390_lowcore.system_timer += S390_lowcore.last_update_timer - S390_lowcore.exit_timer;
S390_lowcore.user_timer += S390_lowcore.exit_timer - S390_lowcore.sys_enter_timer;
S390_lowcore.last_update_timer = S390_lowcore.sys_enter_timer;
struct lowcore *lc = get_lowcore();
lc->system_timer += lc->last_update_timer - lc->exit_timer;
lc->user_timer += lc->exit_timer - lc->sys_enter_timer;
lc->last_update_timer = lc->sys_enter_timer;
}
static inline void update_timer_mcck(void)
{
S390_lowcore.system_timer += S390_lowcore.last_update_timer - S390_lowcore.exit_timer;
S390_lowcore.user_timer += S390_lowcore.exit_timer - S390_lowcore.mcck_enter_timer;
S390_lowcore.last_update_timer = S390_lowcore.mcck_enter_timer;
struct lowcore *lc = get_lowcore();
lc->system_timer += lc->last_update_timer - lc->exit_timer;
lc->user_timer += lc->exit_timer - lc->mcck_enter_timer;
lc->last_update_timer = lc->mcck_enter_timer;
}
#endif /* _S390_VTIME_H */

1
arch/s390/kernel/asm-offsets.c
View File

@ -63,6 +63,7 @@ int main(void)
OFFSET(__SF_SIE_REASON, stack_frame, sie_reason);
OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags);
OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys);
OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce);
DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame));
BLANK();
OFFSET(__SFUSER_BACKCHAIN, stack_frame_user, back_chain);

12
arch/s390/kernel/diag.c
View File

@ -185,6 +185,8 @@ int diag14(unsigned long rx, unsigned long ry1, unsigned long subcode)
}
EXPORT_SYMBOL(diag14);
#define DIAG204_BUSY_RC 8
static inline int __diag204(unsigned long *subcode, unsigned long size, void *addr)
{
union register_pair rp = { .even = *subcode, .odd = size };
@ -215,16 +217,18 @@ int diag204(unsigned long subcode, unsigned long size, void *addr)
{
if (addr) {
if (WARN_ON_ONCE(!is_vmalloc_addr(addr)))
return -1;
return -EINVAL;
if (WARN_ON_ONCE(!IS_ALIGNED((unsigned long)addr, PAGE_SIZE)))
return -1;
return -EINVAL;
}
if ((subcode & DIAG204_SUBCODE_MASK) == DIAG204_SUBC_STIB4)
addr = (void *)pfn_to_phys(vmalloc_to_pfn(addr));
diag_stat_inc(DIAG_STAT_X204);
size = __diag204(&subcode, size, addr);
if (subcode)
return -1;
if (subcode == DIAG204_BUSY_RC)
return -EBUSY;
else if (subcode)
return -EOPNOTSUPP;
return size;
}
EXPORT_SYMBOL(diag204);

8
arch/s390/kernel/dumpstack.c
View File

@ -61,28 +61,28 @@ static bool in_task_stack(unsigned long sp, struct task_struct *task,
static bool in_irq_stack(unsigned long sp, struct stack_info *info)
{
unsigned long stack = S390_lowcore.async_stack - STACK_INIT_OFFSET;
unsigned long stack = get_lowcore()->async_stack - STACK_INIT_OFFSET;
return in_stack(sp, info, STACK_TYPE_IRQ, stack);
}
static bool in_nodat_stack(unsigned long sp, struct stack_info *info)
{
unsigned long stack = S390_lowcore.nodat_stack - STACK_INIT_OFFSET;
unsigned long stack = get_lowcore()->nodat_stack - STACK_INIT_OFFSET;
return in_stack(sp, info, STACK_TYPE_NODAT, stack);
}
static bool in_mcck_stack(unsigned long sp, struct stack_info *info)
{
unsigned long stack = S390_lowcore.mcck_stack - STACK_INIT_OFFSET;
unsigned long stack = get_lowcore()->mcck_stack - STACK_INIT_OFFSET;
return in_stack(sp, info, STACK_TYPE_MCCK, stack);
}
static bool in_restart_stack(unsigned long sp, struct stack_info *info)
{
unsigned long stack = S390_lowcore.restart_stack - STACK_INIT_OFFSET;
unsigned long stack = get_lowcore()->restart_stack - STACK_INIT_OFFSET;
return in_stack(sp, info, STACK_TYPE_RESTART, stack);
}

36
arch/s390/kernel/early.c
View File

@ -72,7 +72,7 @@ static void __init reset_tod_clock(void)
memset(&tod_clock_base, 0, sizeof(tod_clock_base));
tod_clock_base.tod = TOD_UNIX_EPOCH;
S390_lowcore.last_update_clock = TOD_UNIX_EPOCH;
get_lowcore()->last_update_clock = TOD_UNIX_EPOCH;
}
/*
@ -99,7 +99,7 @@ static noinline __init void detect_machine_type(void)
/* Check current-configuration-level */
if (stsi(NULL, 0, 0, 0) <= 2) {
S390_lowcore.machine_flags |= MACHINE_FLAG_LPAR;
get_lowcore()->machine_flags |= MACHINE_FLAG_LPAR;
return;
}
/* Get virtual-machine cpu information. */
@ -108,9 +108,9 @@ static noinline __init void detect_machine_type(void)
/* Detect known hypervisors */
if (!memcmp(vmms->vm[0].cpi, "\xd2\xe5\xd4", 3))
S390_lowcore.machine_flags |= MACHINE_FLAG_KVM;
get_lowcore()->machine_flags |= MACHINE_FLAG_KVM;
else if (!memcmp(vmms->vm[0].cpi, "\xa9\x61\xe5\xd4", 4))
S390_lowcore.machine_flags |= MACHINE_FLAG_VM;
get_lowcore()->machine_flags |= MACHINE_FLAG_VM;
}
/* Remove leading, trailing and double whitespace. */
@ -166,7 +166,7 @@ static __init void setup_topology(void)
if (!test_facility(11))
return;
S390_lowcore.machine_flags |= MACHINE_FLAG_TOPOLOGY;
get_lowcore()->machine_flags |= MACHINE_FLAG_TOPOLOGY;
for (max_mnest = 6; max_mnest > 1; max_mnest--) {
if (stsi(&sysinfo_page, 15, 1, max_mnest) == 0)
break;
@ -186,8 +186,8 @@ static noinline __init void setup_lowcore_early(void)
psw.addr = (unsigned long)early_pgm_check_handler;
psw.mask = PSW_KERNEL_BITS;
S390_lowcore.program_new_psw = psw;
S390_lowcore.preempt_count = INIT_PREEMPT_COUNT;
get_lowcore()->program_new_psw = psw;
get_lowcore()->preempt_count = INIT_PREEMPT_COUNT;
}
static noinline __init void setup_facility_list(void)
@ -211,43 +211,43 @@ static __init void detect_diag9c(void)
EX_TABLE(0b,1b)
: "=d" (rc) : "0" (-EOPNOTSUPP), "d" (cpu_address) : "cc");
if (!rc)
S390_lowcore.machine_flags |= MACHINE_FLAG_DIAG9C;
get_lowcore()->machine_flags |= MACHINE_FLAG_DIAG9C;
}
static __init void detect_machine_facilities(void)
{
if (test_facility(8)) {
S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1;
get_lowcore()->machine_flags |= MACHINE_FLAG_EDAT1;
system_ctl_set_bit(0, CR0_EDAT_BIT);
}
if (test_facility(78))
S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2;
get_lowcore()->machine_flags |= MACHINE_FLAG_EDAT2;
if (test_facility(3))
S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE;
get_lowcore()->machine_flags |= MACHINE_FLAG_IDTE;
if (test_facility(50) && test_facility(73)) {
S390_lowcore.machine_flags |= MACHINE_FLAG_TE;
get_lowcore()->machine_flags |= MACHINE_FLAG_TE;
system_ctl_set_bit(0, CR0_TRANSACTIONAL_EXECUTION_BIT);
}
if (test_facility(51))
S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC;
get_lowcore()->machine_flags |= MACHINE_FLAG_TLB_LC;
if (test_facility(129))
system_ctl_set_bit(0, CR0_VECTOR_BIT);
if (test_facility(130))
S390_lowcore.machine_flags |= MACHINE_FLAG_NX;
get_lowcore()->machine_flags |= MACHINE_FLAG_NX;
if (test_facility(133))
S390_lowcore.machine_flags |= MACHINE_FLAG_GS;
get_lowcore()->machine_flags |= MACHINE_FLAG_GS;
if (test_facility(139) && (tod_clock_base.tod >> 63)) {
/* Enabled signed clock comparator comparisons */
S390_lowcore.machine_flags |= MACHINE_FLAG_SCC;
get_lowcore()->machine_flags |= MACHINE_FLAG_SCC;
clock_comparator_max = -1ULL >> 1;
system_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SIGN_BIT);
}
if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) {
S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO;
get_lowcore()->machine_flags |= MACHINE_FLAG_PCI_MIO;
/* the control bit is set during PCI initialization */
}
if (test_facility(194))
S390_lowcore.machine_flags |= MACHINE_FLAG_RDP;
get_lowcore()->machine_flags |= MACHINE_FLAG_RDP;
}
static inline void save_vector_registers(void)

8
arch/s390/kernel/entry.S
View File

@ -179,6 +179,7 @@ SYM_FUNC_END(__switch_to_asm)
* %r2 pointer to sie control block phys
* %r3 pointer to sie control block virt
* %r4 guest register save area
* %r5 guest asce
*/
SYM_FUNC_START(__sie64a)
stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
@ -186,15 +187,12 @@ SYM_FUNC_START(__sie64a)
stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical..
stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses
stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area
stg %r5,__SF_SIE_GUEST_ASCE(%r15) # save guest asce
xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
lmg %r0,%r13,0(%r4) # load guest gprs 0-13
lg %r14,__LC_GMAP # get gmap pointer
ltgr %r14,%r14
jz .Lsie_gmap
oi __LC_CPU_FLAGS+7,_CIF_SIE
lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
.Lsie_gmap:
lctlg %c1,%c1,__SF_SIE_GUEST_ASCE(%r15) # load primary asce
lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
tm __SIE_PROG20+3(%r14),3 # last exit...

11
arch/s390/kernel/idle.c
View File

@ -24,6 +24,7 @@ static DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
void account_idle_time_irq(void)
{
struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
struct lowcore *lc = get_lowcore();
unsigned long idle_time;
u64 cycles_new[8];
int i;
@ -34,13 +35,13 @@ void account_idle_time_irq(void)
this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]);
}
idle_time = S390_lowcore.int_clock - idle->clock_idle_enter;
idle_time = lc->int_clock - idle->clock_idle_enter;
S390_lowcore.steal_timer += idle->clock_idle_enter - S390_lowcore.last_update_clock;
S390_lowcore.last_update_clock = S390_lowcore.int_clock;
lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock;
lc->last_update_clock = lc->int_clock;
S390_lowcore.system_timer += S390_lowcore.last_update_timer - idle->timer_idle_enter;
S390_lowcore.last_update_timer = S390_lowcore.sys_enter_timer;
lc->system_timer += lc->last_update_timer - idle->timer_idle_enter;
lc->last_update_timer = lc->sys_enter_timer;
/* Account time spent with enabled wait psw loaded as idle time. */
WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time);

18
arch/s390/kernel/irq.c
View File

@ -100,8 +100,8 @@ static const struct irq_class irqclass_sub_desc[] = {
static void do_IRQ(struct pt_regs *regs, int irq)
{
if (tod_after_eq(S390_lowcore.int_clock,
S390_lowcore.clock_comparator))
if (tod_after_eq(get_lowcore()->int_clock,
get_lowcore()->clock_comparator))
/* Serve timer interrupts first. */
clock_comparator_work();
generic_handle_irq(irq);
@ -111,7 +111,7 @@ static int on_async_stack(void)
{
unsigned long frame = current_frame_address();
return ((S390_lowcore.async_stack ^ frame) & ~(THREAD_SIZE - 1)) == 0;
return ((get_lowcore()->async_stack ^ frame) & ~(THREAD_SIZE - 1)) == 0;
}
static void do_irq_async(struct pt_regs *regs, int irq)
@ -119,7 +119,7 @@ static void do_irq_async(struct pt_regs *regs, int irq)
if (on_async_stack()) {
do_IRQ(regs, irq);
} else {
call_on_stack(2, S390_lowcore.async_stack, void, do_IRQ,
call_on_stack(2, get_lowcore()->async_stack, void, do_IRQ,
struct pt_regs *, regs, int, irq);
}
}
@ -153,8 +153,8 @@ void noinstr do_io_irq(struct pt_regs *regs)
set_cpu_flag(CIF_NOHZ_DELAY);
do {
regs->tpi_info = S390_lowcore.tpi_info;
if (S390_lowcore.tpi_info.adapter_IO)
regs->tpi_info = get_lowcore()->tpi_info;
if (get_lowcore()->tpi_info.adapter_IO)
do_irq_async(regs, THIN_INTERRUPT);
else
do_irq_async(regs, IO_INTERRUPT);
@ -183,9 +183,9 @@ void noinstr do_ext_irq(struct pt_regs *regs)
current->thread.last_break = regs->last_break;
}
regs->int_code = S390_lowcore.ext_int_code_addr;
regs->int_parm = S390_lowcore.ext_params;
regs->int_parm_long = S390_lowcore.ext_params2;
regs->int_code = get_lowcore()->ext_int_code_addr;
regs->int_parm = get_lowcore()->ext_params;
regs->int_parm_long = get_lowcore()->ext_params2;
from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT);
if (from_idle)

4
arch/s390/kernel/machine_kexec.c
View File

@ -52,7 +52,7 @@ static void __do_machine_kdump(void *data)
purgatory = (purgatory_t)image->start;
/* store_status() saved the prefix register to lowcore */
prefix = (unsigned long) S390_lowcore.prefixreg_save_area;
prefix = (unsigned long)get_lowcore()->prefixreg_save_area;
/* Now do the reset */
s390_reset_system();
@ -91,7 +91,7 @@ static noinline void __machine_kdump(void *image)
continue;
}
/* Store status of the boot CPU */
mcesa = __va(S390_lowcore.mcesad & MCESA_ORIGIN_MASK);
mcesa = __va(get_lowcore()->mcesad & MCESA_ORIGIN_MASK);
if (cpu_has_vx())
save_vx_regs((__vector128 *) mcesa->vector_save_area);
if (MACHINE_HAS_GS) {

31
arch/s390/kernel/nmi.c
View File

@ -117,6 +117,7 @@ static __always_inline char *u64_to_hex(char *dest, u64 val)
static notrace void s390_handle_damage(void)
{
struct lowcore *lc = get_lowcore();
union ctlreg0 cr0, cr0_new;
char message[100];
psw_t psw_save;
@ -125,7 +126,7 @@ static notrace void s390_handle_damage(void)
smp_emergency_stop();
diag_amode31_ops.diag308_reset();
ptr = nmi_puts(message, "System stopped due to unrecoverable machine check, code: 0x");
u64_to_hex(ptr, S390_lowcore.mcck_interruption_code);
u64_to_hex(ptr, lc->mcck_interruption_code);
/*
* Disable low address protection and make machine check new PSW a
@ -135,17 +136,17 @@ static notrace void s390_handle_damage(void)
cr0_new = cr0;
cr0_new.lap = 0;
local_ctl_load(0, &cr0_new.reg);
psw_save = S390_lowcore.mcck_new_psw;
psw_bits(S390_lowcore.mcck_new_psw).io = 0;
psw_bits(S390_lowcore.mcck_new_psw).ext = 0;
psw_bits(S390_lowcore.mcck_new_psw).wait = 1;
psw_save = lc->mcck_new_psw;
psw_bits(lc->mcck_new_psw).io = 0;
psw_bits(lc->mcck_new_psw).ext = 0;
psw_bits(lc->mcck_new_psw).wait = 1;
sclp_emergency_printk(message);
/*
* Restore machine check new PSW and control register 0 to original
* values. This makes possible system dump analysis easier.
*/
S390_lowcore.mcck_new_psw = psw_save;
lc->mcck_new_psw = psw_save;
local_ctl_load(0, &cr0.reg);
disabled_wait();
while (1);
@ -226,7 +227,7 @@ static bool notrace nmi_registers_valid(union mci mci)
/*
* Set the clock comparator register to the next expected value.
*/
set_clock_comparator(S390_lowcore.clock_comparator);
set_clock_comparator(get_lowcore()->clock_comparator);
if (!mci.gr || !mci.fp || !mci.fc)
return false;
/*
@ -252,7 +253,7 @@ static bool notrace nmi_registers_valid(union mci mci)
* check handling must take care of this. The host values are saved by
* KVM and are not affected.
*/
cr2.reg = S390_lowcore.cregs_save_area[2];
cr2.reg = get_lowcore()->cregs_save_area[2];
if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST))
return false;
if (!mci.ms || !mci.pm || !mci.ia)
@ -278,11 +279,10 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs)
sie_page = container_of(sie_block, struct sie_page, sie_block);
mcck_backup = &sie_page->mcck_info;
mcck_backup->mcic = S390_lowcore.mcck_interruption_code &
mcck_backup->mcic = get_lowcore()->mcck_interruption_code &
~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE);
mcck_backup->ext_damage_code = S390_lowcore.external_damage_code;
mcck_backup->failing_storage_address
= S390_lowcore.failing_storage_address;
mcck_backup->ext_damage_code = get_lowcore()->external_damage_code;
mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address;
}
NOKPROBE_SYMBOL(s390_backup_mcck_info);
@ -302,6 +302,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
static int ipd_count;
static DEFINE_SPINLOCK(ipd_lock);
static unsigned long long last_ipd;
struct lowcore *lc = get_lowcore();
struct mcck_struct *mcck;
unsigned long long tmp;
irqentry_state_t irq_state;
@ -314,7 +315,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
if (user_mode(regs))
update_timer_mcck();
inc_irq_stat(NMI_NMI);
mci.val = S390_lowcore.mcck_interruption_code;
mci.val = lc->mcck_interruption_code;
mcck = this_cpu_ptr(&cpu_mcck);
/*
@ -382,9 +383,9 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
}
if (mci.ed && mci.ec) {
/* External damage */
if (S390_lowcore.external_damage_code & (1U << ED_STP_SYNC))
if (lc->external_damage_code & (1U << ED_STP_SYNC))
mcck->stp_queue |= stp_sync_check();
if (S390_lowcore.external_damage_code & (1U << ED_STP_ISLAND))
if (lc->external_damage_code & (1U << ED_STP_ISLAND))
mcck->stp_queue |= stp_island_check();
mcck_pending = 1;
}

2
arch/s390/kernel/perf_cpum_sf.c
View File

@ -1022,7 +1022,7 @@ static void cpumsf_pmu_enable(struct pmu *pmu)
}
/* Load current program parameter */
lpp(&S390_lowcore.lpp);
lpp(&get_lowcore()->lpp);
debug_sprintf_event(sfdbg, 6, "%s: es %i cs %i ed %i cd %i "
"interval %#lx tear %#lx dear %#lx\n", __func__,

189
arch/s390/kernel/perf_pai_crypto.c
View File

@ -36,8 +36,8 @@ struct paicrypt_map {
struct pai_userdata *save; /* Page to store no-zero counters */
unsigned int active_events; /* # of PAI crypto users */
refcount_t refcnt; /* Reference count mapped buffers */
enum paievt_mode mode; /* Type of event */
struct perf_event *event; /* Perf event for sampling */
struct list_head syswide_list; /* List system-wide sampling events */
};
struct paicrypt_mapptr {
@ -84,20 +84,16 @@ static DEFINE_MUTEX(pai_reserve_mutex);
/* Adjust usage counters and remove allocated memory when all users are
* gone.
*/
static void paicrypt_event_destroy(struct perf_event *event)
static void paicrypt_event_destroy_cpu(struct perf_event *event, int cpu)
{
struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr,
event->cpu);
struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, cpu);
struct paicrypt_map *cpump = mp->mapptr;
static_branch_dec(&pai_key);
mutex_lock(&pai_reserve_mutex);
debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d"
" mode %d refcnt %u\n", __func__,
event->attr.config, event->cpu,
cpump->active_events, cpump->mode,
debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d "
"refcnt %u\n", __func__, event->attr.config,
event->cpu, cpump->active_events,
refcount_read(&cpump->refcnt));
free_page(PAI_SAVE_AREA(event));
if (refcount_dec_and_test(&cpump->refcnt)) {
debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
__func__, (unsigned long)cpump->page,
@ -111,6 +107,23 @@ static void paicrypt_event_destroy(struct perf_event *event)
mutex_unlock(&pai_reserve_mutex);
}
static void paicrypt_event_destroy(struct perf_event *event)
{
int cpu;
static_branch_dec(&pai_key);
free_page(PAI_SAVE_AREA(event));
if (event->cpu == -1) {
struct cpumask *mask = PAI_CPU_MASK(event);
for_each_cpu(cpu, mask)
paicrypt_event_destroy_cpu(event, cpu);
kfree(mask);
} else {
paicrypt_event_destroy_cpu(event, event->cpu);
}
}
static u64 paicrypt_getctr(unsigned long *page, int nr, bool kernel)
{
if (kernel)
@ -156,23 +169,15 @@ static u64 paicrypt_getall(struct perf_event *event)
return sum;
}
/* Used to avoid races in checking concurrent access of counting and
* sampling for crypto events
*
* Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
* allowed and when this event is running, no counting event is allowed.
* Several counting events are allowed in parallel, but no sampling event
* is allowed while one (or more) counting events are running.
*
/* Check concurrent access of counting and sampling for crypto events.
* This function is called in process context and it is save to block.
* When the event initialization functions fails, no other call back will
* be invoked.
*
* Allocate the memory for the event.
*/
static struct paicrypt_map *paicrypt_busy(struct perf_event *event)
static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu)
{
struct perf_event_attr *a = &event->attr;
struct paicrypt_map *cpump = NULL;
struct paicrypt_mapptr *mp;
int rc;
@ -185,7 +190,7 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event)
goto unlock;
/* Allocate node for this event */
mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu);
mp = per_cpu_ptr(paicrypt_root.mapptr, cpu);
cpump = mp->mapptr;
if (!cpump) { /* Paicrypt_map allocated? */
cpump = kzalloc(sizeof(*cpump), GFP_KERNEL);
@ -193,25 +198,9 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event)
rc = -ENOMEM;
goto free_root;
}
INIT_LIST_HEAD(&cpump->syswide_list);
}
if (a->sample_period) { /* Sampling requested */
if (cpump->mode != PAI_MODE_NONE)
rc = -EBUSY; /* ... sampling/counting active */
} else { /* Counting requested */
if (cpump->mode == PAI_MODE_SAMPLING)
rc = -EBUSY; /* ... and sampling active */
}
/*
* This error case triggers when there is a conflict:
* Either sampling requested and counting already active, or visa
* versa. Therefore the struct paicrypto_map for this CPU is
* needed or the error could not have occurred. Only adjust root
* node refcount.
*/
if (rc)
goto free_root;
/* Allocate memory for counter page and counter extraction.
* Only the first counting event has to allocate a page.
*/
@ -235,26 +224,58 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event)
/* Set mode and reference count */
rc = 0;
refcount_set(&cpump->refcnt, 1);
cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING;
mp->mapptr = cpump;
debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d"
" mode %d refcnt %u page %#lx save %p rc %d\n",
__func__, a->sample_period, cpump->active_events,
cpump->mode, refcount_read(&cpump->refcnt),
debug_sprintf_event(cfm_dbg, 5, "%s users %d refcnt %u page %#lx "
"save %p rc %d\n", __func__, cpump->active_events,
refcount_read(&cpump->refcnt),
(unsigned long)cpump->page, cpump->save, rc);
goto unlock;
free_paicrypt_map:
/* Undo memory allocation */
kfree(cpump);
mp->mapptr = NULL;
free_root:
paicrypt_root_free();
unlock:
mutex_unlock(&pai_reserve_mutex);
return rc ? ERR_PTR(rc) : cpump;
}
static int paicrypt_event_init_all(struct perf_event *event)
{
struct paicrypt_map *cpump;
struct cpumask *maskptr;
int cpu, rc = -ENOMEM;
maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL);
if (!maskptr)
goto out;
for_each_online_cpu(cpu) {
cpump = paicrypt_busy(event, cpu);
if (IS_ERR(cpump)) {
for_each_cpu(cpu, maskptr)
paicrypt_event_destroy_cpu(event, cpu);
kfree(maskptr);
rc = PTR_ERR(cpump);
goto out;
}
cpumask_set_cpu(cpu, maskptr);
}
/*
* On error all cpumask are freed and all events have been destroyed.
* Save of which CPUs data structures have been allocated for.
* Release them in paicrypt_event_destroy call back function
* for this event.
*/
PAI_CPU_MASK(event) = maskptr;
rc = 0;
out:
return rc;
}
/* Might be called on different CPU than the one the event is intended for. */
static int paicrypt_event_init(struct perf_event *event)
{
@ -269,10 +290,7 @@ static int paicrypt_event_init(struct perf_event *event)
if (a->config < PAI_CRYPTO_BASE ||
a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
return -EINVAL;
/* Allow only CPU wide operation, no process context for now. */
if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)
return -ENOENT;
/* Allow only CRYPTO_ALL for sampling. */
/* Allow only CRYPTO_ALL for sampling */
if (a->sample_period && a->config != PAI_CRYPTO_BASE)
return -EINVAL;
/* Get a page to store last counter values for sampling */
@ -284,13 +302,17 @@ static int paicrypt_event_init(struct perf_event *event)
}
}
cpump = paicrypt_busy(event);
if (IS_ERR(cpump)) {
if (event->cpu >= 0) {
cpump = paicrypt_busy(event, event->cpu);
if (IS_ERR(cpump))
rc = PTR_ERR(cpump);
} else {
rc = paicrypt_event_init_all(event);
}
if (rc) {
free_page(PAI_SAVE_AREA(event));
rc = PTR_ERR(cpump);
goto out;
}
event->destroy = paicrypt_event_destroy;
if (a->sample_period) {
@ -331,8 +353,14 @@ static void paicrypt_start(struct perf_event *event, int flags)
sum = paicrypt_getall(event); /* Get current value */
local64_set(&event->hw.prev_count, sum);
} else { /* Sampling */
cpump->event = event;
perf_sched_cb_inc(event->pmu);
memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE);
/* Enable context switch callback for system-wide sampling */
if (!(event->attach_state & PERF_ATTACH_TASK)) {
list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);
perf_sched_cb_inc(event->pmu);
} else {
cpump->event = event;
}
}
}
@ -344,7 +372,7 @@ static int paicrypt_add(struct perf_event *event, int flags)
if (++cpump->active_events == 1) {
ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET;
WRITE_ONCE(S390_lowcore.ccd, ccd);
WRITE_ONCE(get_lowcore()->ccd, ccd);
local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
}
if (flags & PERF_EF_START)
@ -353,6 +381,7 @@ static int paicrypt_add(struct perf_event *event, int flags)
return 0;
}
static void paicrypt_have_sample(struct perf_event *, struct paicrypt_map *);
static void paicrypt_stop(struct perf_event *event, int flags)
{
struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
@ -361,8 +390,13 @@ static void paicrypt_stop(struct perf_event *event, int flags)
if (!event->attr.sample_period) { /* Counting */
paicrypt_read(event);
} else { /* Sampling */
perf_sched_cb_dec(event->pmu);
cpump->event = NULL;
if (!(event->attach_state & PERF_ATTACH_TASK)) {
perf_sched_cb_dec(event->pmu);
list_del(PAI_SWLIST(event));
} else {
paicrypt_have_sample(event, cpump);
cpump->event = NULL;
}
}
event->hw.state = PERF_HES_STOPPED;
}
@ -375,7 +409,7 @@ static void paicrypt_del(struct perf_event *event, int flags)
paicrypt_stop(event, PERF_EF_UPDATE);
if (--cpump->active_events == 0) {
local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
WRITE_ONCE(S390_lowcore.ccd, 0);
WRITE_ONCE(get_lowcore()->ccd, 0);
}
}
@ -455,23 +489,30 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump,
}
/* Check if there is data to be saved on schedule out of a task. */
static int paicrypt_have_sample(void)
static void paicrypt_have_sample(struct perf_event *event,
struct paicrypt_map *cpump)
{
size_t rawsize;
if (!event) /* No event active */
return;
rawsize = paicrypt_copy(cpump->save, cpump->page,
(unsigned long *)PAI_SAVE_AREA(event),
event->attr.exclude_user,
event->attr.exclude_kernel);
if (rawsize) /* No incremented counters */
paicrypt_push_sample(rawsize, cpump, event);
}
/* Check if there is data to be saved on schedule out of a task. */
static void paicrypt_have_samples(void)
{
struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
struct paicrypt_map *cpump = mp->mapptr;
struct perf_event *event = cpump->event;
size_t rawsize;
int rc = 0;
struct perf_event *event;
if (!event) /* No event active */
return 0;
rawsize = paicrypt_copy(cpump->save, cpump->page,
(unsigned long *)PAI_SAVE_AREA(event),
cpump->event->attr.exclude_user,
cpump->event->attr.exclude_kernel);
if (rawsize) /* No incremented counters */
rc = paicrypt_push_sample(rawsize, cpump, event);
return rc;
list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)
paicrypt_have_sample(event, cpump);
}
/* Called on schedule-in and schedule-out. No access to event structure,
@ -480,10 +521,10 @@ static int paicrypt_have_sample(void)
static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
{
/* We started with a clean page on event installation. So read out
* results on schedule_out and if page was dirty, clear values.
* results on schedule_out and if page was dirty, save old values.
*/
if (!sched_in)
paicrypt_have_sample();
paicrypt_have_samples();
}
/* Attribute definitions for paicrypt interface. As with other CPU
@ -527,7 +568,7 @@ static const struct attribute_group *paicrypt_attr_groups[] = {
/* Performance monitoring unit for mapped counters */
static struct pmu paicrypt = {
.task_ctx_nr = perf_invalid_context,
.task_ctx_nr = perf_hw_context,
.event_init = paicrypt_event_init,
.add = paicrypt_add,
.del = paicrypt_del,

146
arch/s390/kernel/perf_pai_ext.c
View File

@ -47,11 +47,11 @@ struct paiext_cb { /* PAI extension 1 control block */
struct paiext_map {
unsigned long *area; /* Area for CPU to store counters */
struct pai_userdata *save; /* Area to store non-zero counters */
enum paievt_mode mode; /* Type of event */
unsigned int active_events; /* # of PAI Extension users */
refcount_t refcnt;
struct perf_event *event; /* Perf event for sampling */
struct paiext_cb *paiext_cb; /* PAI extension control block area */
struct list_head syswide_list; /* List system-wide sampling events */
};
struct paiext_mapptr {
@ -70,6 +70,8 @@ static void paiext_root_free(void)
free_percpu(paiext_root.mapptr);
paiext_root.mapptr = NULL;
}
debug_sprintf_event(paiext_dbg, 5, "%s root.refcount %d\n", __func__,
refcount_read(&paiext_root.refcnt));
}
/* On initialization of first event also allocate per CPU data dynamically.
@ -115,20 +117,34 @@ static void paiext_free(struct paiext_mapptr *mp)
}
/* Release the PMU if event is the last perf event */
static void paiext_event_destroy(struct perf_event *event)
static void paiext_event_destroy_cpu(struct perf_event *event, int cpu)
{
struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu);
struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, cpu);
struct paiext_map *cpump = mp->mapptr;
free_page(PAI_SAVE_AREA(event));
mutex_lock(&paiext_reserve_mutex);
if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */
paiext_free(mp);
paiext_root_free();
mutex_unlock(&paiext_reserve_mutex);
debug_sprintf_event(paiext_dbg, 4, "%s cpu %d mapptr %p\n", __func__,
event->cpu, mp->mapptr);
}
static void paiext_event_destroy(struct perf_event *event)
{
int cpu;
free_page(PAI_SAVE_AREA(event));
if (event->cpu == -1) {
struct cpumask *mask = PAI_CPU_MASK(event);
for_each_cpu(cpu, mask)
paiext_event_destroy_cpu(event, cpu);
kfree(mask);
} else {
paiext_event_destroy_cpu(event, event->cpu);
}
debug_sprintf_event(paiext_dbg, 4, "%s cpu %d\n", __func__,
event->cpu);
}
/* Used to avoid races in checking concurrent access of counting and
@ -145,19 +161,18 @@ static void paiext_event_destroy(struct perf_event *event)
*
* Allocate the memory for the event.
*/
static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event)
static int paiext_alloc_cpu(struct perf_event *event, int cpu)
{
struct paiext_mapptr *mp;
struct paiext_map *cpump;
int rc;
mutex_lock(&paiext_reserve_mutex);
rc = paiext_root_alloc();
if (rc)
goto unlock;
mp = per_cpu_ptr(paiext_root.mapptr, event->cpu);
mp = per_cpu_ptr(paiext_root.mapptr, cpu);
cpump = mp->mapptr;
if (!cpump) { /* Paiext_map allocated? */
rc = -ENOMEM;
@ -185,24 +200,13 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event)
paiext_free(mp);
goto undo;
}
INIT_LIST_HEAD(&cpump->syswide_list);
refcount_set(&cpump->refcnt, 1);
cpump->mode = a->sample_period ? PAI_MODE_SAMPLING
: PAI_MODE_COUNTING;
rc = 0;
} else {
/* Multiple invocation, check what is active.
* Supported are multiple counter events or only one sampling
* event concurrently at any one time.
*/
if (cpump->mode == PAI_MODE_SAMPLING ||
(cpump->mode == PAI_MODE_COUNTING && a->sample_period)) {
rc = -EBUSY;
goto undo;
}
refcount_inc(&cpump->refcnt);
}
rc = 0;
undo:
if (rc) {
/* Error in allocation of event, decrement anchor. Since
@ -217,6 +221,38 @@ unlock:
return rc;
}
static int paiext_alloc(struct perf_event *event)
{
struct cpumask *maskptr;
int cpu, rc = -ENOMEM;
maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL);
if (!maskptr)
goto out;
for_each_online_cpu(cpu) {
rc = paiext_alloc_cpu(event, cpu);
if (rc) {
for_each_cpu(cpu, maskptr)
paiext_event_destroy_cpu(event, cpu);
kfree(maskptr);
goto out;
}
cpumask_set_cpu(cpu, maskptr);
}
/*
* On error all cpumask are freed and all events have been destroyed.
* Save of which CPUs data structures have been allocated for.
* Release them in paicrypt_event_destroy call back function
* for this event.
*/
PAI_CPU_MASK(event) = maskptr;
rc = 0;
out:
return rc;
}
/* The PAI extension 1 control block supports up to 128 entries. Return
* the index within PAIE1_CB given the event number. Also validate event
* number.
@ -246,9 +282,6 @@ static int paiext_event_init(struct perf_event *event)
rc = paiext_event_valid(event);
if (rc)
return rc;
/* Allow only CPU wide operation, no process context for now. */
if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)
return -ENOENT;
/* Allow only event NNPA_ALL for sampling. */
if (a->sample_period && a->config != PAI_NNPA_BASE)
return -EINVAL;
@ -262,7 +295,10 @@ static int paiext_event_init(struct perf_event *event)
return -ENOMEM;
}
rc = paiext_alloc(a, event);
if (event->cpu >= 0)
rc = paiext_alloc_cpu(event, event->cpu);
else
rc = paiext_alloc(event);
if (rc) {
free_page(PAI_SAVE_AREA(event));
return rc;
@ -334,8 +370,15 @@ static void paiext_start(struct perf_event *event, int flags)
sum = paiext_getall(event); /* Get current value */
local64_set(&event->hw.prev_count, sum);
} else { /* Sampling */
cpump->event = event;
perf_sched_cb_inc(event->pmu);
memcpy((void *)PAI_SAVE_AREA(event), cpump->area,
PAIE1_CTRBLOCK_SZ);
/* Enable context switch callback for system-wide sampling */
if (!(event->attach_state & PERF_ATTACH_TASK)) {
list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);
perf_sched_cb_inc(event->pmu);
} else {
cpump->event = event;
}
}
}
@ -346,12 +389,10 @@ static int paiext_add(struct perf_event *event, int flags)
struct paiext_cb *pcb = cpump->paiext_cb;
if (++cpump->active_events == 1) {
S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb);
get_lowcore()->aicd = virt_to_phys(cpump->paiext_cb);
pcb->acc = virt_to_phys(cpump->area) | 0x1;
/* Enable CPU instruction lookup for PAIE1 control block */
local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT);
debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n",
__func__, S390_lowcore.aicd, pcb->acc);
}
if (flags & PERF_EF_START)
paiext_start(event, PERF_EF_RELOAD);
@ -359,6 +400,7 @@ static int paiext_add(struct perf_event *event, int flags)
return 0;
}
static void paiext_have_sample(struct perf_event *, struct paiext_map *);
static void paiext_stop(struct perf_event *event, int flags)
{
struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
@ -367,8 +409,13 @@ static void paiext_stop(struct perf_event *event, int flags)
if (!event->attr.sample_period) { /* Counting */
paiext_read(event);
} else { /* Sampling */
perf_sched_cb_dec(event->pmu);
cpump->event = NULL;
if (!(event->attach_state & PERF_ATTACH_TASK)) {
list_del(PAI_SWLIST(event));
perf_sched_cb_dec(event->pmu);
} else {
paiext_have_sample(event, cpump);
cpump->event = NULL;
}
}
event->hw.state = PERF_HES_STOPPED;
}
@ -384,9 +431,7 @@ static void paiext_del(struct perf_event *event, int flags)
/* Disable CPU instruction lookup for PAIE1 control block */
local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT);
pcb->acc = 0;
S390_lowcore.aicd = 0;
debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n",
__func__, S390_lowcore.aicd, pcb->acc);
get_lowcore()->aicd = 0;
}
}
@ -470,21 +515,28 @@ static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump,
}
/* Check if there is data to be saved on schedule out of a task. */
static int paiext_have_sample(void)
static void paiext_have_sample(struct perf_event *event,
struct paiext_map *cpump)
{
struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
struct paiext_map *cpump = mp->mapptr;
struct perf_event *event = cpump->event;
size_t rawsize;
int rc = 0;
if (!event)
return 0;
return;
rawsize = paiext_copy(cpump->save, cpump->area,
(unsigned long *)PAI_SAVE_AREA(event));
if (rawsize) /* Incremented counters */
rc = paiext_push_sample(rawsize, cpump, event);
return rc;
paiext_push_sample(rawsize, cpump, event);
}
/* Check if there is data to be saved on schedule out of a task. */
static void paiext_have_samples(void)
{
struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
struct paiext_map *cpump = mp->mapptr;
struct perf_event *event;
list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)
paiext_have_sample(event, cpump);
}
/* Called on schedule-in and schedule-out. No access to event structure,
@ -493,10 +545,10 @@ static int paiext_have_sample(void)
static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
{
/* We started with a clean page on event installation. So read out
* results on schedule_out and if page was dirty, clear values.
* results on schedule_out and if page was dirty, save old values.
*/
if (!sched_in)
paiext_have_sample();
paiext_have_samples();
}
/* Attribute definitions for pai extension1 interface. As with other CPU
@ -542,7 +594,7 @@ static const struct attribute_group *paiext_attr_groups[] = {
/* Performance monitoring unit for mapped counters */
static struct pmu paiext = {
.task_ctx_nr = perf_invalid_context,
.task_ctx_nr = perf_hw_context,
.event_init = paiext_event_init,
.add = paiext_add,
.del = paiext_del,

6
arch/s390/kernel/process.c
View File

@ -71,10 +71,10 @@ void flush_thread(void)
void arch_setup_new_exec(void)
{
if (S390_lowcore.current_pid != current->pid) {
S390_lowcore.current_pid = current->pid;
if (get_lowcore()->current_pid != current->pid) {
get_lowcore()->current_pid = current->pid;
if (test_facility(40))
lpp(&S390_lowcore.lpp);
lpp(&get_lowcore()->lpp);
}
}

24
arch/s390/kernel/setup.c
View File

@ -421,16 +421,16 @@ static void __init setup_lowcore(void)
lc->clock_comparator = clock_comparator_max;
lc->current_task = (unsigned long)&init_task;
lc->lpp = LPP_MAGIC;
lc->machine_flags = S390_lowcore.machine_flags;
lc->preempt_count = S390_lowcore.preempt_count;
lc->machine_flags = get_lowcore()->machine_flags;
lc->preempt_count = get_lowcore()->preempt_count;
nmi_alloc_mcesa_early(&lc->mcesad);
lc->sys_enter_timer = S390_lowcore.sys_enter_timer;
lc->exit_timer = S390_lowcore.exit_timer;
lc->user_timer = S390_lowcore.user_timer;
lc->system_timer = S390_lowcore.system_timer;
lc->steal_timer = S390_lowcore.steal_timer;
lc->last_update_timer = S390_lowcore.last_update_timer;
lc->last_update_clock = S390_lowcore.last_update_clock;
lc->sys_enter_timer = get_lowcore()->sys_enter_timer;
lc->exit_timer = get_lowcore()->exit_timer;
lc->user_timer = get_lowcore()->user_timer;
lc->system_timer = get_lowcore()->system_timer;
lc->steal_timer = get_lowcore()->steal_timer;
lc->last_update_timer = get_lowcore()->last_update_timer;
lc->last_update_clock = get_lowcore()->last_update_clock;
/*
* Allocate the global restart stack which is the same for
* all CPUs in case *one* of them does a PSW restart.
@ -439,7 +439,7 @@ static void __init setup_lowcore(void)
lc->mcck_stack = stack_alloc_early() + STACK_INIT_OFFSET;
lc->async_stack = stack_alloc_early() + STACK_INIT_OFFSET;
lc->nodat_stack = stack_alloc_early() + STACK_INIT_OFFSET;
lc->kernel_stack = S390_lowcore.kernel_stack;
lc->kernel_stack = get_lowcore()->kernel_stack;
/*
* Set up PSW restart to call ipl.c:do_restart(). Copy the relevant
* restart data to the absolute zero lowcore. This is necessary if
@ -455,8 +455,8 @@ static void __init setup_lowcore(void)
lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);
lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);
lc->preempt_count = PREEMPT_DISABLED;
lc->kernel_asce = S390_lowcore.kernel_asce;
lc->user_asce = S390_lowcore.user_asce;
lc->kernel_asce = get_lowcore()->kernel_asce;
lc->user_asce = get_lowcore()->user_asce;
system_ctlreg_init_save_area(lc);
abs_lc = get_abs_lowcore();

88
arch/s390/kernel/smp.c
View File

@ -74,8 +74,6 @@ enum {
CPU_STATE_CONFIGURED,
};
static DEFINE_PER_CPU(struct cpu *, cpu_device);
struct pcpu {
unsigned long ec_mask; /* bit mask for ec_xxx functions */
unsigned long ec_clk; /* sigp timestamp for ec_xxx */
@ -203,7 +201,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
mcck_stack = stack_alloc();
if (!lc || !nodat_stack || !async_stack || !mcck_stack)
goto out;
memcpy(lc, &S390_lowcore, 512);
memcpy(lc, get_lowcore(), 512);
memset((char *) lc + 512, 0, sizeof(*lc) - 512);
lc->async_stack = async_stack + STACK_INIT_OFFSET;
lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;
@ -265,9 +263,9 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
lc->spinlock_lockval = arch_spin_lockval(cpu);
lc->spinlock_index = 0;
lc->percpu_offset = __per_cpu_offset[cpu];
lc->kernel_asce = S390_lowcore.kernel_asce;
lc->kernel_asce = get_lowcore()->kernel_asce;
lc->user_asce = s390_invalid_asce;
lc->machine_flags = S390_lowcore.machine_flags;
lc->machine_flags = get_lowcore()->machine_flags;
lc->user_timer = lc->system_timer =
lc->steal_timer = lc->avg_steal_timer = 0;
abs_lc = get_abs_lowcore();
@ -407,7 +405,7 @@ void smp_call_ipl_cpu(void (*func)(void *), void *data)
struct lowcore *lc = lowcore_ptr[0];
if (pcpu_devices[0].address == stap())
lc = &S390_lowcore;
lc = get_lowcore();
pcpu_delegate(&pcpu_devices[0], func, data,
lc->nodat_stack);
@ -719,8 +717,6 @@ static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
}
}
static int smp_add_present_cpu(int cpu);
static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
bool configured, bool early)
{
@ -744,7 +740,7 @@ static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
pcpu->state = CPU_STATE_STANDBY;
smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
set_cpu_present(cpu, true);
if (!early && smp_add_present_cpu(cpu) != 0)
if (!early && arch_register_cpu(cpu))
set_cpu_present(cpu, false);
else
nr++;
@ -831,9 +827,6 @@ void __init smp_detect_cpus(void)
s_cpus += smp_cpu_mtid + 1;
}
pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
/* Add CPUs present at boot */
__smp_rescan_cpus(info, true);
memblock_free(info, sizeof(*info));
}
@ -842,15 +835,16 @@ void __init smp_detect_cpus(void)
*/
static void smp_start_secondary(void *cpuvoid)
{
struct lowcore *lc = get_lowcore();
int cpu = raw_smp_processor_id();
S390_lowcore.last_update_clock = get_tod_clock();
S390_lowcore.restart_stack = (unsigned long)restart_stack;
S390_lowcore.restart_fn = (unsigned long)do_restart;
S390_lowcore.restart_data = 0;
S390_lowcore.restart_source = -1U;
S390_lowcore.restart_flags = 0;
restore_access_regs(S390_lowcore.access_regs_save_area);
lc->last_update_clock = get_tod_clock();
lc->restart_stack = (unsigned long)restart_stack;
lc->restart_fn = (unsigned long)do_restart;
lc->restart_data = 0;
lc->restart_source = -1U;
lc->restart_flags = 0;
restore_access_regs(lc->access_regs_save_area);
cpu_init();
rcutree_report_cpu_starting(cpu);
init_cpu_timer();
@ -973,6 +967,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
panic("Couldn't request external interrupt 0x1202");
system_ctl_set_bit(0, 13);
smp_rescan_cpus(true);
}
void __init smp_prepare_boot_cpu(void)
@ -981,16 +976,18 @@ void __init smp_prepare_boot_cpu(void)
WARN_ON(!cpu_present(0) || !cpu_online(0));
pcpu->state = CPU_STATE_CONFIGURED;
S390_lowcore.percpu_offset = __per_cpu_offset[0];
get_lowcore()->percpu_offset = __per_cpu_offset[0];
smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
}
void __init smp_setup_processor_id(void)
{
struct lowcore *lc = get_lowcore();
pcpu_devices[0].address = stap();
S390_lowcore.cpu_nr = 0;
S390_lowcore.spinlock_lockval = arch_spin_lockval(0);
S390_lowcore.spinlock_index = 0;
lc->cpu_nr = 0;
lc->spinlock_lockval = arch_spin_lockval(0);
lc->spinlock_index = 0;
}
/*
@ -1108,35 +1105,34 @@ static struct attribute_group cpu_online_attr_group = {
static int smp_cpu_online(unsigned int cpu)
{
struct device *s = &per_cpu(cpu_device, cpu)->dev;
struct cpu *c = &per_cpu(cpu_devices, cpu);
return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
return sysfs_create_group(&c->dev.kobj, &cpu_online_attr_group);
}
static int smp_cpu_pre_down(unsigned int cpu)
{
struct device *s = &per_cpu(cpu_device, cpu)->dev;
struct cpu *c = &per_cpu(cpu_devices, cpu);
sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
sysfs_remove_group(&c->dev.kobj, &cpu_online_attr_group);
return 0;
}
static int smp_add_present_cpu(int cpu)
bool arch_cpu_is_hotpluggable(int cpu)
{
struct device *s;
struct cpu *c;
return !!cpu;
}
int arch_register_cpu(int cpu)
{
struct cpu *c = &per_cpu(cpu_devices, cpu);
int rc;
c = kzalloc(sizeof(*c), GFP_KERNEL);
if (!c)
return -ENOMEM;
per_cpu(cpu_device, cpu) = c;
s = &c->dev;
c->hotpluggable = !!cpu;
c->hotpluggable = arch_cpu_is_hotpluggable(cpu);
rc = register_cpu(c, cpu);
if (rc)
goto out;
rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
rc = sysfs_create_group(&c->dev.kobj, &cpu_common_attr_group);
if (rc)
goto out_cpu;
rc = topology_cpu_init(c);
@ -1145,14 +1141,14 @@ static int smp_add_present_cpu(int cpu)
return 0;
out_topology:
sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
sysfs_remove_group(&c->dev.kobj, &cpu_common_attr_group);
out_cpu:
unregister_cpu(c);
out:
return rc;
}
int __ref smp_rescan_cpus(void)
int __ref smp_rescan_cpus(bool early)
{
struct sclp_core_info *info;
int nr;
@ -1161,7 +1157,7 @@ int __ref smp_rescan_cpus(void)
if (!info)
return -ENOMEM;
smp_get_core_info(info, 0);
nr = __smp_rescan_cpus(info, false);
nr = __smp_rescan_cpus(info, early);
kfree(info);
if (nr)
topology_schedule_update();
@ -1178,7 +1174,7 @@ static ssize_t __ref rescan_store(struct device *dev,
rc = lock_device_hotplug_sysfs();
if (rc)
return rc;
rc = smp_rescan_cpus();
rc = smp_rescan_cpus(false);
unlock_device_hotplug();
return rc ? rc : count;
}
@ -1187,7 +1183,7 @@ static DEVICE_ATTR_WO(rescan);
static int __init s390_smp_init(void)
{
struct device *dev_root;
int cpu, rc = 0;
int rc;
dev_root = bus_get_dev_root(&cpu_subsys);
if (dev_root) {
@ -1196,17 +1192,9 @@ static int __init s390_smp_init(void)
if (rc)
return rc;
}
for_each_present_cpu(cpu) {
rc = smp_add_present_cpu(cpu);
if (rc)
goto out;
}
rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
smp_cpu_online, smp_cpu_pre_down);
rc = rc <= 0 ? rc : 0;
out:
return rc;
}
subsys_initcall(s390_smp_init);

95
arch/s390/kernel/sthyi.c
View File

@ -300,34 +300,57 @@ static struct diag204_x_part_block *lpar_cpu_inf(struct lpar_cpu_inf *part_inf,
return (struct diag204_x_part_block *)&block->cpus[i];
}
static void fill_diag(struct sthyi_sctns *sctns)
static void *diag204_get_data(bool diag204_allow_busy)
{
int i, r, pages;
bool this_lpar;
unsigned long subcode;
void *diag204_buf;
int pages, rc;
subcode = DIAG204_SUBC_RSI;
subcode |= DIAG204_INFO_EXT;
pages = diag204(subcode, 0, NULL);
if (pages < 0)
return ERR_PTR(pages);
if (pages == 0)
return ERR_PTR(-ENODATA);
diag204_buf = __vmalloc_node(array_size(pages, PAGE_SIZE),
PAGE_SIZE, GFP_KERNEL, NUMA_NO_NODE,
__builtin_return_address(0));
if (!diag204_buf)
return ERR_PTR(-ENOMEM);
subcode = DIAG204_SUBC_STIB7;
subcode |= DIAG204_INFO_EXT;
if (diag204_has_bif() && diag204_allow_busy)
subcode |= DIAG204_BIF_BIT;
rc = diag204(subcode, pages, diag204_buf);
if (rc < 0) {
vfree(diag204_buf);
return ERR_PTR(rc);
}
return diag204_buf;
}
static bool is_diag204_cached(struct sthyi_sctns *sctns)
{
/*
* Check if validity bits are set when diag204 data
* is gathered.
*/
if (sctns->par.infpval1)
return true;
return false;
}
static void fill_diag(struct sthyi_sctns *sctns, void *diag204_buf)
{
int i;
bool this_lpar;
void *diag224_buf = NULL;
struct diag204_x_info_blk_hdr *ti_hdr;
struct diag204_x_part_block *part_block;
struct diag204_x_phys_block *phys_block;
struct lpar_cpu_inf lpar_inf = {};
/* Errors are handled through the validity bits in the response. */
pages = diag204((unsigned long)DIAG204_SUBC_RSI |
(unsigned long)DIAG204_INFO_EXT, 0, NULL);
if (pages <= 0)
return;
diag204_buf = __vmalloc_node(array_size(pages, PAGE_SIZE),
PAGE_SIZE, GFP_KERNEL, NUMA_NO_NODE,
__builtin_return_address(0));
if (!diag204_buf)
return;
r = diag204((unsigned long)DIAG204_SUBC_STIB7 |
(unsigned long)DIAG204_INFO_EXT, pages, diag204_buf);
if (r < 0)
goto out;
diag224_buf = (void *)__get_free_page(GFP_KERNEL | GFP_DMA);
if (!diag224_buf || diag224(diag224_buf))
goto out;
@ -392,7 +415,6 @@ static void fill_diag(struct sthyi_sctns *sctns)
out:
free_page((unsigned long)diag224_buf);
vfree(diag204_buf);
}
static int sthyi(u64 vaddr, u64 *rc)
@ -414,19 +436,31 @@ static int sthyi(u64 vaddr, u64 *rc)
static int fill_dst(void *dst, u64 *rc)
{
void *diag204_buf;
struct sthyi_sctns *sctns = (struct sthyi_sctns *)dst;
/*
* If the facility is on, we don't want to emulate the instruction.
* We ask the hypervisor to provide the data.
*/
if (test_facility(74))
if (test_facility(74)) {
memset(dst, 0, PAGE_SIZE);
return sthyi((u64)dst, rc);
}
/*
* When emulating, if diag204 returns BUSY don't reset dst buffer
* and use cached data.
*/
*rc = 0;
diag204_buf = diag204_get_data(is_diag204_cached(sctns));
if (IS_ERR(diag204_buf))
return PTR_ERR(diag204_buf);
memset(dst, 0, PAGE_SIZE);
fill_hdr(sctns);
fill_stsi(sctns);
fill_diag(sctns);
*rc = 0;
fill_diag(sctns, diag204_buf);
vfree(diag204_buf);
return 0;
}
@ -445,11 +479,14 @@ static int sthyi_update_cache(u64 *rc)
{
int r;
memset(sthyi_cache.info, 0, PAGE_SIZE);
r = fill_dst(sthyi_cache.info, rc);
if (r)
return r;
sthyi_cache.end = jiffies + CACHE_VALID_JIFFIES;
if (r == 0) {
sthyi_cache.end = jiffies + CACHE_VALID_JIFFIES;
} else if (r == -EBUSY) {
/* mark as expired and return 0 to keep using cached data */
sthyi_cache.end = jiffies - 1;
r = 0;
}
return r;
}

4
arch/s390/kernel/syscall.c
View File

@ -124,8 +124,8 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap)
{
add_random_kstack_offset();
enter_from_user_mode(regs);
regs->psw = S390_lowcore.svc_old_psw;
regs->int_code = S390_lowcore.svc_int_code;
regs->psw = get_lowcore()->svc_old_psw;
regs->int_code = get_lowcore()->svc_int_code;
update_timer_sys();
if (static_branch_likely(&cpu_has_bear))
current->thread.last_break = regs->last_break;

22
arch/s390/kernel/time.c
View File

@ -131,7 +131,7 @@ void clock_comparator_work(void)
{
struct clock_event_device *cd;
S390_lowcore.clock_comparator = clock_comparator_max;
get_lowcore()->clock_comparator = clock_comparator_max;
cd = this_cpu_ptr(&comparators);
cd->event_handler(cd);
}
@ -139,8 +139,8 @@ void clock_comparator_work(void)
static int s390_next_event(unsigned long delta,
struct clock_event_device *evt)
{
S390_lowcore.clock_comparator = get_tod_clock() + delta;
set_clock_comparator(S390_lowcore.clock_comparator);
get_lowcore()->clock_comparator = get_tod_clock() + delta;
set_clock_comparator(get_lowcore()->clock_comparator);
return 0;
}
@ -153,8 +153,8 @@ void init_cpu_timer(void)
struct clock_event_device *cd;
int cpu;
S390_lowcore.clock_comparator = clock_comparator_max;
set_clock_comparator(S390_lowcore.clock_comparator);
get_lowcore()->clock_comparator = clock_comparator_max;
set_clock_comparator(get_lowcore()->clock_comparator);
cpu = smp_processor_id();
cd = &per_cpu(comparators, cpu);
@ -184,8 +184,8 @@ static void clock_comparator_interrupt(struct ext_code ext_code,
unsigned long param64)
{
inc_irq_stat(IRQEXT_CLK);
if (S390_lowcore.clock_comparator == clock_comparator_max)
set_clock_comparator(S390_lowcore.clock_comparator);
if (get_lowcore()->clock_comparator == clock_comparator_max)
set_clock_comparator(get_lowcore()->clock_comparator);
}
static void stp_timing_alert(struct stp_irq_parm *);
@ -408,12 +408,12 @@ static void clock_sync_global(long delta)
static void clock_sync_local(long delta)
{
/* Add the delta to the clock comparator. */
if (S390_lowcore.clock_comparator != clock_comparator_max) {
S390_lowcore.clock_comparator += delta;
set_clock_comparator(S390_lowcore.clock_comparator);
if (get_lowcore()->clock_comparator != clock_comparator_max) {
get_lowcore()->clock_comparator += delta;
set_clock_comparator(get_lowcore()->clock_comparator);
}
/* Adjust the last_update_clock time-stamp. */
S390_lowcore.last_update_clock += delta;
get_lowcore()->last_update_clock += delta;
}
/* Single threaded workqueue used for stp sync events */

8
arch/s390/kernel/topology.c
View File

@ -320,16 +320,10 @@ static int __arch_update_cpu_topology(void)
int arch_update_cpu_topology(void)
{
struct device *dev;
int cpu, rc;
int rc;
rc = __arch_update_cpu_topology();
on_each_cpu(__arch_update_dedicated_flag, NULL, 0);
for_each_online_cpu(cpu) {
dev = get_cpu_device(cpu);
if (dev)
kobject_uevent(&dev->kobj, KOBJ_CHANGE);
}
return rc;
}

28
arch/s390/kernel/traps.c
View File

@ -288,15 +288,16 @@ static void __init test_monitor_call(void)
void __init trap_init(void)
{
struct lowcore *lc = get_lowcore();
unsigned long flags;
struct ctlreg cr0;
local_irq_save(flags);
cr0 = local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT);
psw_bits(S390_lowcore.external_new_psw).mcheck = 1;
psw_bits(S390_lowcore.program_new_psw).mcheck = 1;
psw_bits(S390_lowcore.svc_new_psw).mcheck = 1;
psw_bits(S390_lowcore.io_new_psw).mcheck = 1;
psw_bits(lc->external_new_psw).mcheck = 1;
psw_bits(lc->program_new_psw).mcheck = 1;
psw_bits(lc->svc_new_psw).mcheck = 1;
psw_bits(lc->io_new_psw).mcheck = 1;
local_ctl_load(0, &cr0);
local_irq_restore(flags);
local_mcck_enable();
@ -307,11 +308,12 @@ static void (*pgm_check_table[128])(struct pt_regs *regs);
void noinstr __do_pgm_check(struct pt_regs *regs)
{
unsigned int trapnr;
struct lowcore *lc = get_lowcore();
irqentry_state_t state;
unsigned int trapnr;
regs->int_code = S390_lowcore.pgm_int_code;
regs->int_parm_long = S390_lowcore.trans_exc_code;
regs->int_code = lc->pgm_int_code;
regs->int_parm_long = lc->trans_exc_code;
state = irqentry_enter(regs);
@ -324,19 +326,19 @@ void noinstr __do_pgm_check(struct pt_regs *regs)
current->thread.last_break = regs->last_break;
}
if (S390_lowcore.pgm_code & 0x0200) {
if (lc->pgm_code & 0x0200) {
/* transaction abort */
current->thread.trap_tdb = S390_lowcore.pgm_tdb;
current->thread.trap_tdb = lc->pgm_tdb;
}
if (S390_lowcore.pgm_code & PGM_INT_CODE_PER) {
if (lc->pgm_code & PGM_INT_CODE_PER) {
if (user_mode(regs)) {
struct per_event *ev = &current->thread.per_event;
set_thread_flag(TIF_PER_TRAP);
ev->address = S390_lowcore.per_address;
ev->cause = S390_lowcore.per_code_combined;
ev->paid = S390_lowcore.per_access_id;
ev->address = lc->per_address;
ev->cause = lc->per_code_combined;
ev->paid = lc->per_access_id;
} else {
/* PER event in kernel is kprobes */
__arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER);

207
arch/s390/kernel/uv.c
View File

@ -110,7 +110,7 @@ EXPORT_SYMBOL_GPL(uv_pin_shared);
*
* @paddr: Absolute host address of page to be destroyed
*/
static int uv_destroy_page(unsigned long paddr)
static int uv_destroy(unsigned long paddr)
{
struct uv_cb_cfs uvcb = {
.header.cmd = UVC_CMD_DESTR_SEC_STOR,
@ -131,28 +131,40 @@ static int uv_destroy_page(unsigned long paddr)
}
/*
* The caller must already hold a reference to the page
* The caller must already hold a reference to the folio
*/
int uv_destroy_owned_page(unsigned long paddr)
int uv_destroy_folio(struct folio *folio)
{
struct page *page = phys_to_page(paddr);
int rc;
get_page(page);
rc = uv_destroy_page(paddr);
/* See gmap_make_secure(): large folios cannot be secure */
if (unlikely(folio_test_large(folio)))
return 0;
folio_get(folio);
rc = uv_destroy(folio_to_phys(folio));
if (!rc)
clear_bit(PG_arch_1, &page->flags);
put_page(page);
clear_bit(PG_arch_1, &folio->flags);
folio_put(folio);
return rc;
}
/*
* The present PTE still indirectly holds a folio reference through the mapping.
*/
int uv_destroy_pte(pte_t pte)
{
VM_WARN_ON(!pte_present(pte));
return uv_destroy_folio(pfn_folio(pte_pfn(pte)));
}
/*
* Requests the Ultravisor to encrypt a guest page and make it
* accessible to the host for paging (export).
*
* @paddr: Absolute host address of page to be exported
*/
int uv_convert_from_secure(unsigned long paddr)
static int uv_convert_from_secure(unsigned long paddr)
{
struct uv_cb_cfs uvcb = {
.header.cmd = UVC_CMD_CONV_FROM_SEC_STOR,
@ -166,21 +178,33 @@ int uv_convert_from_secure(unsigned long paddr)
}
/*
* The caller must already hold a reference to the page
* The caller must already hold a reference to the folio.
*/
int uv_convert_owned_from_secure(unsigned long paddr)
static int uv_convert_from_secure_folio(struct folio *folio)
{
struct page *page = phys_to_page(paddr);
int rc;
get_page(page);
rc = uv_convert_from_secure(paddr);
/* See gmap_make_secure(): large folios cannot be secure */
if (unlikely(folio_test_large(folio)))
return 0;
folio_get(folio);
rc = uv_convert_from_secure(folio_to_phys(folio));
if (!rc)
clear_bit(PG_arch_1, &page->flags);
put_page(page);
clear_bit(PG_arch_1, &folio->flags);
folio_put(folio);
return rc;
}
/*
* The present PTE still indirectly holds a folio reference through the mapping.
*/
int uv_convert_from_secure_pte(pte_t pte)
{
VM_WARN_ON(!pte_present(pte));
return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte)));
}
/*
* Calculate the expected ref_count for a folio that would otherwise have no
* further pins. This was cribbed from similar functions in other places in
@ -266,6 +290,36 @@ static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_str
return atomic_read(&mm->context.protected_count) > 1;
}
/*
* Drain LRU caches: the local one on first invocation and the ones of all
* CPUs on successive invocations. Returns "true" on the first invocation.
*/
static bool drain_lru(bool *drain_lru_called)
{
/*
* If we have tried a local drain and the folio refcount
* still does not match our expected safe value, try with a
* system wide drain. This is needed if the pagevecs holding
* the page are on a different CPU.
*/
if (*drain_lru_called) {
lru_add_drain_all();
/* We give up here, don't retry immediately. */
return false;
}
/*
* We are here if the folio refcount does not match the
* expected safe value. The main culprits are usually
* pagevecs. With lru_add_drain() we drain the pagevecs
* on the local CPU so that hopefully the refcount will
* reach the expected safe value.
*/
lru_add_drain();
*drain_lru_called = true;
/* The caller should try again immediately */
return true;
}
/*
* Requests the Ultravisor to make a page accessible to a guest.
* If it's brought in the first time, it will be cleared. If
@ -275,7 +329,7 @@ static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_str
int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb)
{
struct vm_area_struct *vma;
bool local_drain = false;
bool drain_lru_called = false;
spinlock_t *ptelock;
unsigned long uaddr;
struct folio *folio;
@ -308,52 +362,63 @@ again:
goto out;
if (pte_present(*ptep) && !(pte_val(*ptep) & _PAGE_INVALID) && pte_write(*ptep)) {
folio = page_folio(pte_page(*ptep));
rc = -EINVAL;
if (folio_test_large(folio))
goto unlock;
rc = -EAGAIN;
if (folio_trylock(folio)) {
if (folio_test_large(folio)) {
rc = -E2BIG;
} else if (folio_trylock(folio)) {
if (should_export_before_import(uvcb, gmap->mm))
uv_convert_from_secure(PFN_PHYS(folio_pfn(folio)));
rc = make_folio_secure(folio, uvcb);
folio_unlock(folio);
}
/*
* Once we drop the PTL, the folio may get unmapped and
* freed immediately. We need a temporary reference.
*/
if (rc == -EAGAIN || rc == -E2BIG)
folio_get(folio);
}
unlock:
pte_unmap_unlock(ptep, ptelock);
out:
mmap_read_unlock(gmap->mm);
if (rc == -EAGAIN) {
switch (rc) {
case -E2BIG:
folio_lock(folio);
rc = split_folio(folio);
folio_unlock(folio);
folio_put(folio);
switch (rc) {
case 0:
/* Splitting succeeded, try again immediately. */
goto again;
case -EAGAIN:
/* Additional folio references. */
if (drain_lru(&drain_lru_called))
goto again;
return -EAGAIN;
case -EBUSY:
/* Unexpected race. */
return -EAGAIN;
}
WARN_ON_ONCE(1);
return -ENXIO;
case -EAGAIN:
/*
* If we are here because the UVC returned busy or partial
* completion, this is just a useless check, but it is safe.
*/
folio_wait_writeback(folio);
} else if (rc == -EBUSY) {
/*
* If we have tried a local drain and the folio refcount
* still does not match our expected safe value, try with a
* system wide drain. This is needed if the pagevecs holding
* the page are on a different CPU.
*/
if (local_drain) {
lru_add_drain_all();
/* We give up here, and let the caller try again */
return -EAGAIN;
}
/*
* We are here if the folio refcount does not match the
* expected safe value. The main culprits are usually
* pagevecs. With lru_add_drain() we drain the pagevecs
* on the local CPU so that hopefully the refcount will
* reach the expected safe value.
*/
lru_add_drain();
local_drain = true;
/* And now we try again immediately after draining */
goto again;
} else if (rc == -ENXIO) {
folio_put(folio);
return -EAGAIN;
case -EBUSY:
/* Additional folio references. */
if (drain_lru(&drain_lru_called))
goto again;
return -EAGAIN;
case -ENXIO:
if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE))
return -EFAULT;
return -EAGAIN;
@ -388,6 +453,7 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr)
{
struct vm_area_struct *vma;
unsigned long uaddr;
struct folio *folio;
struct page *page;
int rc;
@ -411,7 +477,8 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr)
page = follow_page(vma, uaddr, FOLL_WRITE | FOLL_GET);
if (IS_ERR_OR_NULL(page))
goto out;
rc = uv_destroy_owned_page(page_to_phys(page));
folio = page_folio(page);
rc = uv_destroy_folio(folio);
/*
* Fault handlers can race; it is possible that two CPUs will fault
* on the same secure page. One CPU can destroy the page, reboot,
@ -422,8 +489,8 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr)
* we instead try to export the page.
*/
if (rc)
rc = uv_convert_owned_from_secure(page_to_phys(page));
put_page(page);
rc = uv_convert_from_secure_folio(folio);
folio_put(folio);
out:
mmap_read_unlock(gmap->mm);
return rc;
@ -431,47 +498,51 @@ out:
EXPORT_SYMBOL_GPL(gmap_destroy_page);
/*
* To be called with the page locked or with an extra reference! This will
* prevent gmap_make_secure from touching the page concurrently. Having 2
* parallel make_page_accessible is fine, as the UV calls will become a
* no-op if the page is already exported.
* To be called with the folio locked or with an extra reference! This will
* prevent gmap_make_secure from touching the folio concurrently. Having 2
* parallel arch_make_folio_accessible is fine, as the UV calls will become a
* no-op if the folio is already exported.
*/
int arch_make_page_accessible(struct page *page)
int arch_make_folio_accessible(struct folio *folio)
{
int rc = 0;
/* Hugepage cannot be protected, so nothing to do */
if (PageHuge(page))
/* See gmap_make_secure(): large folios cannot be secure */
if (unlikely(folio_test_large(folio)))
return 0;
/*
* PG_arch_1 is used in 3 places:
* 1. for kernel page tables during early boot
* 2. for storage keys of huge pages and KVM
* 3. As an indication that this page might be secure. This can
* PG_arch_1 is used in 2 places:
* 1. for storage keys of hugetlb folios and KVM
* 2. As an indication that this small folio might be secure. This can
* overindicate, e.g. we set the bit before calling
* convert_to_secure.
* As secure pages are never huge, all 3 variants can co-exists.
* As secure pages are never large folios, both variants can co-exists.
*/
if (!test_bit(PG_arch_1, &page->flags))
if (!test_bit(PG_arch_1, &folio->flags))
return 0;
rc = uv_pin_shared(page_to_phys(page));
rc = uv_pin_shared(folio_to_phys(folio));
if (!rc) {
clear_bit(PG_arch_1, &page->flags);
clear_bit(PG_arch_1, &folio->flags);
return 0;
}
rc = uv_convert_from_secure(page_to_phys(page));
rc = uv_convert_from_secure(folio_to_phys(folio));
if (!rc) {
clear_bit(PG_arch_1, &page->flags);
clear_bit(PG_arch_1, &folio->flags);
return 0;
}
return rc;
}
EXPORT_SYMBOL_GPL(arch_make_page_accessible);
EXPORT_SYMBOL_GPL(arch_make_folio_accessible);
int arch_make_page_accessible(struct page *page)
{
return arch_make_folio_accessible(page_folio(page));
}
EXPORT_SYMBOL_GPL(arch_make_page_accessible);
#endif
#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM)

82
arch/s390/kernel/vtime.c
View File

@ -35,14 +35,15 @@ static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
static inline void set_vtimer(u64 expires)
{
struct lowcore *lc = get_lowcore();
u64 timer;
asm volatile(
" stpt %0\n" /* Store current cpu timer value */
" spt %1" /* Set new value imm. afterwards */
: "=Q" (timer) : "Q" (expires));
S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
S390_lowcore.last_update_timer = expires;
lc->system_timer += lc->last_update_timer - timer;
lc->last_update_timer = expires;
}
static inline int virt_timer_forward(u64 elapsed)
@ -117,22 +118,23 @@ static void account_system_index_scaled(struct task_struct *p, u64 cputime,
static int do_account_vtime(struct task_struct *tsk)
{
u64 timer, clock, user, guest, system, hardirq, softirq;
struct lowcore *lc = get_lowcore();
timer = S390_lowcore.last_update_timer;
clock = S390_lowcore.last_update_clock;
timer = lc->last_update_timer;
clock = lc->last_update_clock;
asm volatile(
" stpt %0\n" /* Store current cpu timer value */
" stckf %1" /* Store current tod clock value */
: "=Q" (S390_lowcore.last_update_timer),
"=Q" (S390_lowcore.last_update_clock)
: "=Q" (lc->last_update_timer),
"=Q" (lc->last_update_clock)
: : "cc");
clock = S390_lowcore.last_update_clock - clock;
timer -= S390_lowcore.last_update_timer;
clock = lc->last_update_clock - clock;
timer -= lc->last_update_timer;
if (hardirq_count())
S390_lowcore.hardirq_timer += timer;
lc->hardirq_timer += timer;
else
S390_lowcore.system_timer += timer;
lc->system_timer += timer;
/* Update MT utilization calculation */
if (smp_cpu_mtid &&
@ -141,16 +143,16 @@ static int do_account_vtime(struct task_struct *tsk)
/* Calculate cputime delta */
user = update_tsk_timer(&tsk->thread.user_timer,
READ_ONCE(S390_lowcore.user_timer));
READ_ONCE(lc->user_timer));
guest = update_tsk_timer(&tsk->thread.guest_timer,
READ_ONCE(S390_lowcore.guest_timer));
READ_ONCE(lc->guest_timer));
system = update_tsk_timer(&tsk->thread.system_timer,
READ_ONCE(S390_lowcore.system_timer));
READ_ONCE(lc->system_timer));
hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
READ_ONCE(S390_lowcore.hardirq_timer));
READ_ONCE(lc->hardirq_timer));
softirq = update_tsk_timer(&tsk->thread.softirq_timer,
READ_ONCE(S390_lowcore.softirq_timer));
S390_lowcore.steal_timer +=
READ_ONCE(lc->softirq_timer));
lc->steal_timer +=
clock - user - guest - system - hardirq - softirq;
/* Push account value */
@ -176,17 +178,19 @@ static int do_account_vtime(struct task_struct *tsk)
void vtime_task_switch(struct task_struct *prev)
{
struct lowcore *lc = get_lowcore();
do_account_vtime(prev);
prev->thread.user_timer = S390_lowcore.user_timer;
prev->thread.guest_timer = S390_lowcore.guest_timer;
prev->thread.system_timer = S390_lowcore.system_timer;
prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
prev->thread.softirq_timer = S390_lowcore.softirq_timer;
S390_lowcore.user_timer = current->thread.user_timer;
S390_lowcore.guest_timer = current->thread.guest_timer;
S390_lowcore.system_timer = current->thread.system_timer;
S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
S390_lowcore.softirq_timer = current->thread.softirq_timer;
prev->thread.user_timer = lc->user_timer;
prev->thread.guest_timer = lc->guest_timer;
prev->thread.system_timer = lc->system_timer;
prev->thread.hardirq_timer = lc->hardirq_timer;
prev->thread.softirq_timer = lc->softirq_timer;
lc->user_timer = current->thread.user_timer;
lc->guest_timer = current->thread.guest_timer;
lc->system_timer = current->thread.system_timer;
lc->hardirq_timer = current->thread.hardirq_timer;
lc->softirq_timer = current->thread.softirq_timer;
}
/*
@ -196,28 +200,29 @@ void vtime_task_switch(struct task_struct *prev)
*/
void vtime_flush(struct task_struct *tsk)
{
struct lowcore *lc = get_lowcore();
u64 steal, avg_steal;
if (do_account_vtime(tsk))
virt_timer_expire();
steal = S390_lowcore.steal_timer;
avg_steal = S390_lowcore.avg_steal_timer;
steal = lc->steal_timer;
avg_steal = lc->avg_steal_timer;
if ((s64) steal > 0) {
S390_lowcore.steal_timer = 0;
lc->steal_timer = 0;
account_steal_time(cputime_to_nsecs(steal));
avg_steal += steal;
}
S390_lowcore.avg_steal_timer = avg_steal / 2;
lc->avg_steal_timer = avg_steal / 2;
}
static u64 vtime_delta(void)
{
u64 timer = S390_lowcore.last_update_timer;
struct lowcore *lc = get_lowcore();
u64 timer = lc->last_update_timer;
S390_lowcore.last_update_timer = get_cpu_timer();
return timer - S390_lowcore.last_update_timer;
lc->last_update_timer = get_cpu_timer();
return timer - lc->last_update_timer;
}
/*
@ -226,12 +231,13 @@ static u64 vtime_delta(void)
*/
void vtime_account_kernel(struct task_struct *tsk)
{
struct lowcore *lc = get_lowcore();
u64 delta = vtime_delta();
if (tsk->flags & PF_VCPU)
S390_lowcore.guest_timer += delta;
lc->guest_timer += delta;
else
S390_lowcore.system_timer += delta;
lc->system_timer += delta;
virt_timer_forward(delta);
}
@ -241,7 +247,7 @@ void vtime_account_softirq(struct task_struct *tsk)
{
u64 delta = vtime_delta();
S390_lowcore.softirq_timer += delta;
get_lowcore()->softirq_timer += delta;
virt_timer_forward(delta);
}
@ -250,7 +256,7 @@ void vtime_account_hardirq(struct task_struct *tsk)
{
u64 delta = vtime_delta();
S390_lowcore.hardirq_timer += delta;
get_lowcore()->hardirq_timer += delta;
virt_timer_forward(delta);
}

163
arch/s390/kvm/gaccess.c
View File

@ -14,167 +14,10 @@
#include <asm/access-regs.h>
#include <asm/fault.h>
#include <asm/gmap.h>
#include <asm/dat-bits.h>
#include "kvm-s390.h"
#include "gaccess.h"
union asce {
unsigned long val;
struct {
unsigned long origin : 52; /* Region- or Segment-Table Origin */
unsigned long : 2;
unsigned long g : 1; /* Subspace Group Control */
unsigned long p : 1; /* Private Space Control */
unsigned long s : 1; /* Storage-Alteration-Event Control */
unsigned long x : 1; /* Space-Switch-Event Control */
unsigned long r : 1; /* Real-Space Control */
unsigned long : 1;
unsigned long dt : 2; /* Designation-Type Control */
unsigned long tl : 2; /* Region- or Segment-Table Length */
};
};
enum {
ASCE_TYPE_SEGMENT = 0,
ASCE_TYPE_REGION3 = 1,
ASCE_TYPE_REGION2 = 2,
ASCE_TYPE_REGION1 = 3
};
union region1_table_entry {
unsigned long val;
struct {
unsigned long rto: 52;/* Region-Table Origin */
unsigned long : 2;
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 1;
unsigned long tf : 2; /* Region-Second-Table Offset */
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long : 1;
unsigned long tt : 2; /* Table-Type Bits */
unsigned long tl : 2; /* Region-Second-Table Length */
};
};
union region2_table_entry {
unsigned long val;
struct {
unsigned long rto: 52;/* Region-Table Origin */
unsigned long : 2;
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 1;
unsigned long tf : 2; /* Region-Third-Table Offset */
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long : 1;
unsigned long tt : 2; /* Table-Type Bits */
unsigned long tl : 2; /* Region-Third-Table Length */
};
};
struct region3_table_entry_fc0 {
unsigned long sto: 52;/* Segment-Table Origin */
unsigned long : 1;
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 1;
unsigned long tf : 2; /* Segment-Table Offset */
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long cr : 1; /* Common-Region Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long tl : 2; /* Segment-Table Length */
};
struct region3_table_entry_fc1 {
unsigned long rfaa : 33; /* Region-Frame Absolute Address */
unsigned long : 14;
unsigned long av : 1; /* ACCF-Validity Control */
unsigned long acc: 4; /* Access-Control Bits */
unsigned long f : 1; /* Fetch-Protection Bit */
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long iep: 1; /* Instruction-Execution-Protection */
unsigned long : 2;
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long cr : 1; /* Common-Region Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
union region3_table_entry {
unsigned long val;
struct region3_table_entry_fc0 fc0;
struct region3_table_entry_fc1 fc1;
struct {
unsigned long : 53;
unsigned long fc : 1; /* Format-Control */
unsigned long : 4;
unsigned long i : 1; /* Region-Invalid Bit */
unsigned long cr : 1; /* Common-Region Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
};
struct segment_entry_fc0 {
unsigned long pto: 53;/* Page-Table Origin */
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long : 3;
unsigned long i : 1; /* Segment-Invalid Bit */
unsigned long cs : 1; /* Common-Segment Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
struct segment_entry_fc1 {
unsigned long sfaa : 44; /* Segment-Frame Absolute Address */
unsigned long : 3;
unsigned long av : 1; /* ACCF-Validity Control */
unsigned long acc: 4; /* Access-Control Bits */
unsigned long f : 1; /* Fetch-Protection Bit */
unsigned long fc : 1; /* Format-Control */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long iep: 1; /* Instruction-Execution-Protection */
unsigned long : 2;
unsigned long i : 1; /* Segment-Invalid Bit */
unsigned long cs : 1; /* Common-Segment Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
union segment_table_entry {
unsigned long val;
struct segment_entry_fc0 fc0;
struct segment_entry_fc1 fc1;
struct {
unsigned long : 53;
unsigned long fc : 1; /* Format-Control */
unsigned long : 4;
unsigned long i : 1; /* Segment-Invalid Bit */
unsigned long cs : 1; /* Common-Segment Bit */
unsigned long tt : 2; /* Table-Type Bits */
unsigned long : 2;
};
};
enum {
TABLE_TYPE_SEGMENT = 0,
TABLE_TYPE_REGION3 = 1,
TABLE_TYPE_REGION2 = 2,
TABLE_TYPE_REGION1 = 3
};
union page_table_entry {
unsigned long val;
struct {
unsigned long pfra : 52; /* Page-Frame Real Address */
unsigned long z : 1; /* Zero Bit */
unsigned long i : 1; /* Page-Invalid Bit */
unsigned long p : 1; /* DAT-Protection Bit */
unsigned long iep: 1; /* Instruction-Execution-Protection */
unsigned long : 8;
};
};
/*
* vaddress union in order to easily decode a virtual address into its
* region first index, region second index etc. parts.
@ -632,7 +475,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
iep = ctlreg0.iep && test_kvm_facility(vcpu->kvm, 130);
if (asce.r)
goto real_address;
ptr = asce.origin * PAGE_SIZE;
ptr = asce.rsto * PAGE_SIZE;
switch (asce.dt) {
case ASCE_TYPE_REGION1:
if (vaddr.rfx01 > asce.tl)
@ -1379,7 +1222,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
parent = sg->parent;
vaddr.addr = saddr;
asce.val = sg->orig_asce;
ptr = asce.origin * PAGE_SIZE;
ptr = asce.rsto * PAGE_SIZE;
if (asce.r) {
*fake = 1;
ptr = 0;

5
arch/s390/kvm/kvm-s390.c
View File

@ -4080,7 +4080,7 @@ static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
{
/* do not poll with more than halt_poll_max_steal percent of steal time */
if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >=
READ_ONCE(halt_poll_max_steal)) {
vcpu->stat.halt_no_poll_steal++;
return true;
@ -4830,7 +4830,8 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
sizeof(sie_page->pv_grregs));
}
exit_reason = sie64a(vcpu->arch.sie_block,
vcpu->run->s.regs.gprs);
vcpu->run->s.regs.gprs,
gmap_get_enabled()->asce);
if (kvm_s390_pv_cpu_is_protected(vcpu)) {
memcpy(vcpu->run->s.regs.gprs,
sie_page->pv_grregs,

2
arch/s390/kvm/vsie.c
View File

@ -1150,7 +1150,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
vcpu->arch.sie_block->prog0c |= PROG_IN_SIE;
barrier();
if (!kvm_s390_vcpu_sie_inhibited(vcpu))
rc = sie64a(scb_s, vcpu->run->s.regs.gprs);
rc = sie64a(scb_s, vcpu->run->s.regs.gprs, gmap_get_enabled()->asce);
barrier();
vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;

4
arch/s390/lib/spinlock.c
View File

@ -119,7 +119,7 @@ static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
struct spin_wait *node, *next;
int lockval, ix, node_id, tail_id, old, new, owner, count;
ix = S390_lowcore.spinlock_index++;
ix = get_lowcore()->spinlock_index++;
barrier();
lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
node = this_cpu_ptr(&spin_wait[ix]);
@ -205,7 +205,7 @@ static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
}
out:
S390_lowcore.spinlock_index--;
get_lowcore()->spinlock_index--;
}
static inline void arch_spin_lock_classic(arch_spinlock_t *lp)

1
arch/s390/lib/test_kprobes.c
View File

@ -72,4 +72,5 @@ static struct kunit_suite kprobes_test_suite = {
kunit_test_suites(&kprobes_test_suite);
MODULE_DESCRIPTION("KUnit tests for kprobes");
MODULE_LICENSE("GPL");

1
arch/s390/lib/test_modules.c
View File

@ -29,4 +29,5 @@ static struct kunit_suite modules_test_suite = {
kunit_test_suites(&modules_test_suite);
MODULE_DESCRIPTION("KUnit test that modules with many relocations are loaded properly");
MODULE_LICENSE("GPL");

3
arch/s390/lib/test_unwind.c
View File

@ -356,7 +356,7 @@ static noinline int unwindme_func2(struct unwindme *u)
if (u->flags & UWM_SWITCH_STACK) {
local_irq_save(flags);
local_mcck_save(mflags);
rc = call_on_stack(1, S390_lowcore.nodat_stack,
rc = call_on_stack(1, get_lowcore()->nodat_stack,
int, unwindme_func3, struct unwindme *, u);
local_mcck_restore(mflags);
local_irq_restore(flags);
@ -519,4 +519,5 @@ static struct kunit_suite test_unwind_suite = {
kunit_test_suites(&test_unwind_suite);
MODULE_DESCRIPTION("KUnit test for unwind_for_each_frame");
MODULE_LICENSE("GPL");

4
arch/s390/lib/uaccess.c
View File

@ -21,13 +21,13 @@ void debug_user_asce(int exit)
local_ctl_store(1, &cr1);
local_ctl_store(7, &cr7);
if (cr1.val == S390_lowcore.kernel_asce.val && cr7.val == S390_lowcore.user_asce.val)
if (cr1.val == get_lowcore()->kernel_asce.val && cr7.val == get_lowcore()->user_asce.val)
return;
panic("incorrect ASCE on kernel %s\n"
"cr1: %016lx cr7: %016lx\n"
"kernel: %016lx user: %016lx\n",
exit ? "exit" : "entry", cr1.val, cr7.val,
S390_lowcore.kernel_asce.val, S390_lowcore.user_asce.val);
get_lowcore()->kernel_asce.val, get_lowcore()->user_asce.val);
}
#endif /*CONFIG_DEBUG_ENTRY */

1
arch/s390/mm/cmm.c
View File

@ -427,4 +427,5 @@ static void __exit cmm_exit(void)
}
module_exit(cmm_exit);
MODULE_DESCRIPTION("Cooperative memory management interface");
MODULE_LICENSE("GPL");

2
arch/s390/mm/dump_pagetables.c
View File

@ -288,7 +288,7 @@ static int pt_dump_init(void)
* kernel ASCE. We need this to keep the page table walker functions
* from accessing non-existent entries.
*/
max_addr = (S390_lowcore.kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2;
max_addr = (get_lowcore()->kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2;
max_addr = 1UL << (max_addr * 11 + 31);
address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size;
address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31;

33
arch/s390/mm/fault.c
View File

@ -74,7 +74,7 @@ static enum fault_type get_fault_type(struct pt_regs *regs)
return USER_FAULT;
if (!IS_ENABLED(CONFIG_PGSTE))
return KERNEL_FAULT;
gmap = (struct gmap *)S390_lowcore.gmap;
gmap = (struct gmap *)get_lowcore()->gmap;
if (gmap && gmap->asce == regs->cr1)
return GMAP_FAULT;
return KERNEL_FAULT;
@ -182,15 +182,15 @@ static void dump_fault_info(struct pt_regs *regs)
pr_cont("mode while using ");
switch (get_fault_type(regs)) {
case USER_FAULT:
asce = S390_lowcore.user_asce.val;
asce = get_lowcore()->user_asce.val;
pr_cont("user ");
break;
case GMAP_FAULT:
asce = ((struct gmap *)S390_lowcore.gmap)->asce;
asce = ((struct gmap *)get_lowcore()->gmap)->asce;
pr_cont("gmap ");
break;
case KERNEL_FAULT:
asce = S390_lowcore.kernel_asce.val;
asce = get_lowcore()->kernel_asce.val;
pr_cont("kernel ");
break;
default:
@ -351,7 +351,7 @@ lock_mmap:
mmap_read_lock(mm);
gmap = NULL;
if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) {
gmap = (struct gmap *)S390_lowcore.gmap;
gmap = (struct gmap *)get_lowcore()->gmap;
current->thread.gmap_addr = address;
current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE);
current->thread.gmap_int_code = regs->int_code & 0xffff;
@ -433,12 +433,13 @@ error:
handle_fault_error_nolock(regs, 0);
else
do_sigsegv(regs, SEGV_MAPERR);
} else if (fault & VM_FAULT_SIGBUS) {
} else if (fault & (VM_FAULT_SIGBUS | VM_FAULT_HWPOISON)) {
if (!user_mode(regs))
handle_fault_error_nolock(regs, 0);
else
do_sigbus(regs);
} else {
pr_emerg("Unexpected fault flags: %08x\n", fault);
BUG();
}
}
@ -492,6 +493,7 @@ void do_secure_storage_access(struct pt_regs *regs)
unsigned long addr = get_fault_address(regs);
struct vm_area_struct *vma;
struct mm_struct *mm;
struct folio *folio;
struct page *page;
struct gmap *gmap;
int rc;
@ -521,7 +523,7 @@ void do_secure_storage_access(struct pt_regs *regs)
switch (get_fault_type(regs)) {
case GMAP_FAULT:
mm = current->mm;
gmap = (struct gmap *)S390_lowcore.gmap;
gmap = (struct gmap *)get_lowcore()->gmap;
mmap_read_lock(mm);
addr = __gmap_translate(gmap, addr);
mmap_read_unlock(mm);
@ -539,17 +541,18 @@ void do_secure_storage_access(struct pt_regs *regs)
mmap_read_unlock(mm);
break;
}
if (arch_make_page_accessible(page))
folio = page_folio(page);
if (arch_make_folio_accessible(folio))
send_sig(SIGSEGV, current, 0);
put_page(page);
folio_put(folio);
mmap_read_unlock(mm);
break;
case KERNEL_FAULT:
page = phys_to_page(addr);
if (unlikely(!try_get_page(page)))
folio = phys_to_folio(addr);
if (unlikely(!folio_try_get(folio)))
break;
rc = arch_make_page_accessible(page);
put_page(page);
rc = arch_make_folio_accessible(folio);
folio_put(folio);
if (rc)
BUG();
break;
@ -561,7 +564,7 @@ NOKPROBE_SYMBOL(do_secure_storage_access);
void do_non_secure_storage_access(struct pt_regs *regs)
{
struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
struct gmap *gmap = (struct gmap *)get_lowcore()->gmap;
unsigned long gaddr = get_fault_address(regs);
if (WARN_ON_ONCE(get_fault_type(regs) != GMAP_FAULT))
@ -573,7 +576,7 @@ NOKPROBE_SYMBOL(do_non_secure_storage_access);
void do_secure_storage_violation(struct pt_regs *regs)
{
struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
struct gmap *gmap = (struct gmap *)get_lowcore()->gmap;
unsigned long gaddr = get_fault_address(regs);
/*

16
arch/s390/mm/gmap.c
View File

@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(gmap_remove);
*/
void gmap_enable(struct gmap *gmap)
{
S390_lowcore.gmap = (unsigned long) gmap;
get_lowcore()->gmap = (unsigned long)gmap;
}
EXPORT_SYMBOL_GPL(gmap_enable);
@ -297,7 +297,7 @@ EXPORT_SYMBOL_GPL(gmap_enable);
*/
void gmap_disable(struct gmap *gmap)
{
S390_lowcore.gmap = 0UL;
get_lowcore()->gmap = 0UL;
}
EXPORT_SYMBOL_GPL(gmap_disable);
@ -308,7 +308,7 @@ EXPORT_SYMBOL_GPL(gmap_disable);
*/
struct gmap *gmap_get_enabled(void)
{
return (struct gmap *) S390_lowcore.gmap;
return (struct gmap *)get_lowcore()->gmap;
}
EXPORT_SYMBOL_GPL(gmap_get_enabled);
@ -2733,7 +2733,7 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
{
pmd_t *pmd = (pmd_t *)pte;
unsigned long start, end;
struct page *page = pmd_page(*pmd);
struct folio *folio = page_folio(pmd_page(*pmd));
/*
* The write check makes sure we do not set a key on shared
@ -2748,7 +2748,7 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
start = pmd_val(*pmd) & HPAGE_MASK;
end = start + HPAGE_SIZE;
__storage_key_init_range(start, end);
set_bit(PG_arch_1, &page->flags);
set_bit(PG_arch_1, &folio->flags);
cond_resched();
return 0;
}
@ -2841,13 +2841,15 @@ static const struct mm_walk_ops gather_pages_ops = {
*/
void s390_uv_destroy_pfns(unsigned long count, unsigned long *pfns)
{
struct folio *folio;
unsigned long i;
for (i = 0; i < count; i++) {
folio = pfn_folio(pfns[i]);
/* we always have an extra reference */
uv_destroy_owned_page(pfn_to_phys(pfns[i]));
uv_destroy_folio(folio);
/* get rid of the extra reference */
put_page(pfn_to_page(pfns[i]));
folio_put(folio);
cond_resched();
}
}

8
arch/s390/mm/hugetlbpage.c
View File

@ -121,7 +121,7 @@ static inline pte_t __rste_to_pte(unsigned long rste)
static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
{
struct page *page;
struct folio *folio;
unsigned long size, paddr;
if (!mm_uses_skeys(mm) ||
@ -129,16 +129,16 @@ static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
return;
if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
page = pud_page(__pud(rste));
folio = page_folio(pud_page(__pud(rste)));
size = PUD_SIZE;
paddr = rste & PUD_MASK;
} else {
page = pmd_page(__pmd(rste));
folio = page_folio(pmd_page(__pmd(rste)));
size = PMD_SIZE;
paddr = rste & PMD_MASK;
}
if (!test_and_set_bit(PG_arch_1, &page->flags))
if (!test_and_set_bit(PG_arch_1, &folio->flags))
__storage_key_init_range(paddr, paddr + size);
}

3
arch/s390/mm/init.c
View File

@ -62,6 +62,7 @@ EXPORT_SYMBOL(zero_page_mask);
static void __init setup_zero_pages(void)
{
unsigned long total_pages = PHYS_PFN(memblock_phys_mem_size() - memblock_reserved_size());
unsigned int order;
struct page *page;
int i;
@ -70,7 +71,7 @@ static void __init setup_zero_pages(void)
order = 7;
/* Limit number of empty zero pages for small memory sizes */
while (order > 2 && (totalram_pages() >> 10) < (1UL << order))
while (order > 2 && (total_pages >> 10) < (1UL << order))
order--;
empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);

2
arch/s390/mm/pageattr.c
View File

@ -75,7 +75,7 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr,
break;
}
table = (unsigned long *)((unsigned long)old & mask);
crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce.val);
crdte(*old, new, table, dtt, addr, get_lowcore()->kernel_asce.val);
} else if (MACHINE_HAS_IDTE) {
cspg(old, *old, new);
} else {

4
arch/s390/mm/pgalloc.c
View File

@ -66,8 +66,8 @@ static void __crst_table_upgrade(void *arg)
/* change all active ASCEs to avoid the creation of new TLBs */
if (current->active_mm == mm) {
S390_lowcore.user_asce.val = mm->context.asce;
local_ctl_load(7, &S390_lowcore.user_asce);
get_lowcore()->user_asce.val = mm->context.asce;
local_ctl_load(7, &get_lowcore()->user_asce);
}
__tlb_flush_local();
}

2
arch/s390/pci/pci.c
View File

@ -1064,7 +1064,7 @@ char * __init pcibios_setup(char *str)
return NULL;
}
if (!strcmp(str, "nomio")) {
S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO;
get_lowcore()->machine_flags &= ~MACHINE_FLAG_PCI_MIO;
return NULL;
}
if (!strcmp(str, "force_floating")) {

1
drivers/s390/block/dcssblk.c
View File

@ -1032,4 +1032,5 @@ MODULE_PARM_DESC(segments, "Name of DCSS segment(s) to be loaded, "
"the contiguous segments - \n"
"e.g. segments=\"mydcss1,mydcss2:mydcss3,mydcss4(local)\"");
MODULE_DESCRIPTION("S/390 block driver for DCSS memory");
MODULE_LICENSE("GPL");

1
drivers/s390/char/con3270.c
View File

@ -2185,6 +2185,7 @@ con3270_init(void)
console_initcall(con3270_init);
#endif
MODULE_DESCRIPTION("IBM/3270 Driver - tty functions");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CHARDEV_MAJOR(IBM_TTY3270_MAJOR);

1
drivers/s390/char/fs3270.c
View File

@ -559,6 +559,7 @@ static void __exit fs3270_exit(void)
__unregister_chrdev(IBM_FS3270_MAJOR, 0, 1, "fs3270");
}
MODULE_DESCRIPTION("IBM/3270 Driver - fullscreen driver");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CHARDEV_MAJOR(IBM_FS3270_MAJOR);

1
drivers/s390/char/raw3270.c
View File

@ -1341,6 +1341,7 @@ static void raw3270_exit(void)
class_unregister(&class3270);
}
MODULE_DESCRIPTION("IBM/3270 Driver - core functions");
MODULE_LICENSE("GPL");
module_init(raw3270_init);

7
drivers/s390/char/sclp_cmd.c
View File

@ -31,6 +31,9 @@
#include "sclp.h"
#define SCLP_CMDW_ASSIGN_STORAGE 0x000d0001
#define SCLP_CMDW_UNASSIGN_STORAGE 0x000c0001
static void sclp_sync_callback(struct sclp_req *req, void *data)
{
struct completion *completion = data;
@ -225,7 +228,7 @@ static int sclp_assign_storage(u16 rn)
unsigned long long start;
int rc;
rc = do_assign_storage(0x000d0001, rn);
rc = do_assign_storage(SCLP_CMDW_ASSIGN_STORAGE, rn);
if (rc)
return rc;
start = rn2addr(rn);
@ -235,7 +238,7 @@ static int sclp_assign_storage(u16 rn)
static int sclp_unassign_storage(u16 rn)
{
return do_assign_storage(0x000c0001, rn);
return do_assign_storage(SCLP_CMDW_UNASSIGN_STORAGE, rn);
}
struct attach_storage_sccb {

2
drivers/s390/char/sclp_config.c
View File

@ -60,7 +60,7 @@ static void sclp_cpu_capability_notify(struct work_struct *work)
static void __ref sclp_cpu_change_notify(struct work_struct *work)
{
lock_device_hotplug();
smp_rescan_cpus();
smp_rescan_cpus(false);
unlock_device_hotplug();
}

5
drivers/s390/char/sclp_early.c
View File

@ -50,9 +50,10 @@ static void __init sclp_early_facilities_detect(void)
sclp.has_aisi = !!(sccb->fac118 & 0x10);
sclp.has_zpci_lsi = !!(sccb->fac118 & 0x01);
if (sccb->fac85 & 0x02)
S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP;
get_lowcore()->machine_flags |= MACHINE_FLAG_ESOP;
if (sccb->fac91 & 0x40)
S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_GUEST;
get_lowcore()->machine_flags |= MACHINE_FLAG_TLB_GUEST;
sclp.has_diag204_bif = !!(sccb->fac98 & 0x80);
if (sccb->cpuoff > 134) {
sclp.has_diag318 = !!(sccb->byte_134 & 0x80);
sclp.has_diag320 = !!(sccb->byte_134 & 0x04);

12
drivers/s390/char/sclp_early_core.c
View File

@ -38,11 +38,11 @@ void sclp_early_wait_irq(void)
cr0_new.sssm = 1;
local_ctl_load(0, &cr0_new.reg);
psw_ext_save = S390_lowcore.external_new_psw;
psw_ext_save = get_lowcore()->external_new_psw;
psw_mask = __extract_psw();
S390_lowcore.external_new_psw.mask = psw_mask;
get_lowcore()->external_new_psw.mask = psw_mask;
psw_wait.mask = psw_mask | PSW_MASK_EXT | PSW_MASK_WAIT;
S390_lowcore.ext_int_code = 0;
get_lowcore()->ext_int_code = 0;
do {
asm volatile(
@ -53,12 +53,12 @@ void sclp_early_wait_irq(void)
"0:\n"
: [addr] "=&d" (addr),
[psw_wait_addr] "=Q" (psw_wait.addr),
[psw_ext_addr] "=Q" (S390_lowcore.external_new_psw.addr)
[psw_ext_addr] "=Q" (get_lowcore()->external_new_psw.addr)
: [psw_wait] "Q" (psw_wait)
: "cc", "memory");
} while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG);
} while (get_lowcore()->ext_int_code != EXT_IRQ_SERVICE_SIG);
S390_lowcore.external_new_psw = psw_ext_save;
get_lowcore()->external_new_psw = psw_ext_save;
local_ctl_load(0, &cr0.reg);
}

26
drivers/s390/char/sclp_sd.c
View File

@ -9,6 +9,7 @@
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
#include <linux/completion.h>
#include <linux/jiffies.h>
#include <linux/kobject.h>
#include <linux/list.h>
#include <linux/printk.h>
@ -28,6 +29,8 @@
#define SD_DI_CONFIG 3
#define SD_TIMEOUT msecs_to_jiffies(30000)
struct sclp_sd_evbuf {
struct evbuf_header hdr;
u8 eq;
@ -194,6 +197,10 @@ static int sclp_sd_sync(unsigned long page, u8 eq, u8 di, u64 sat, u64 sa,
struct sclp_sd_evbuf *evbuf;
int rc;
if (!sclp_sd_register.sclp_send_mask ||
!sclp_sd_register.sclp_receive_mask)
return -EIO;
sclp_sd_listener_init(&listener, __pa(sccb));
sclp_sd_listener_add(&listener);
@ -230,9 +237,12 @@ static int sclp_sd_sync(unsigned long page, u8 eq, u8 di, u64 sat, u64 sa,
goto out;
}
if (!(evbuf->rflags & 0x80)) {
rc = wait_for_completion_interruptible(&listener.completion);
if (rc)
rc = wait_for_completion_interruptible_timeout(&listener.completion, SD_TIMEOUT);
if (rc == 0)
rc = -ETIME;
if (rc < 0)
goto out;
rc = 0;
evbuf = &listener.evbuf;
}
switch (evbuf->status) {
@ -319,9 +329,15 @@ static int sclp_sd_store_data(struct sclp_sd_data *result, u8 di)
rc = sclp_sd_sync(page, SD_EQ_STORE_DATA, di, asce, (u64) data, &dsize,
&esize);
if (rc) {
/* Cancel running request if interrupted */
if (rc == -ERESTARTSYS)
sclp_sd_sync(page, SD_EQ_HALT, di, 0, 0, NULL, NULL);
/* Cancel running request if interrupted or timed out */
if (rc == -ERESTARTSYS || rc == -ETIME) {
if (sclp_sd_sync(page, SD_EQ_HALT, di, 0, 0, NULL, NULL)) {
pr_warn("Could not stop Store Data request - leaking at least %zu bytes\n",
(size_t)dsize * PAGE_SIZE);
data = NULL;
asce = 0;
}
}
vfree(data);
goto out;
}

2
drivers/s390/cio/qdio_main.c
View File

@ -695,7 +695,7 @@ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
return;
qdio_deliver_irq(irq_ptr);
irq_ptr->last_data_irq_time = S390_lowcore.int_clock;
irq_ptr->last_data_irq_time = get_lowcore()->int_clock;
}
static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,

2
drivers/s390/cio/qdio_thinint.c
View File

@ -99,7 +99,7 @@ static inline u32 clear_shared_ind(void)
static void tiqdio_thinint_handler(struct airq_struct *airq,
struct tpi_info *tpi_info)
{
u64 irq_time = S390_lowcore.int_clock;
u64 irq_time = get_lowcore()->int_clock;
u32 si_used = clear_shared_ind();
struct qdio_irq *irq;

2
drivers/s390/cio/trace.h
View File

@ -169,7 +169,7 @@ TRACE_EVENT(s390_cio_tpi,
else if (addr)
__entry->tpi_info = *addr;
else
__entry->tpi_info = S390_lowcore.tpi_info;
__entry->tpi_info = get_lowcore()->tpi_info;
__entry->cssid = __entry->tpi_info.schid.cssid;
__entry->ssid = __entry->tpi_info.schid.ssid;
__entry->schno = __entry->tpi_info.schid.sch_no;