leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

arm64 updates for 3.20:

Browse Source 
- reimplementation of the virtual remapping of UEFI Runtime Services in
   a way that is stable across kexec
 - emulation of the "setend" instruction for 32-bit tasks (user
   endianness switching trapped in the kernel, SCTLR_EL1.E0E bit set
   accordingly)
 - compat_sys_call_table implemented in C (from asm) and made it a
   constant array together with sys_call_table
 - export CPU cache information via /sys (like other architectures)
 - DMA API implementation clean-up in preparation for IOMMU support
 - macros clean-up for KVM
 - dropped some unnecessary cache+tlb maintenance
 - CONFIG_ARM64_CPU_SUSPEND clean-up
 - defconfig update (CPU_IDLE)
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
 iQIcBAABAgAGBQJU25v3AAoJEGvWsS0AyF7xYjcP/j8ESvs+z0BPgeJ6XREfOnCh
 cp+w/1rJ5BafJ5RRkibrciwTNOIJS4FGMivWyURtoh430lS0Rh7fxZ3Ouna3xjrT
 Nf7AxenWoA8Lo6wHh+FlNUeGk3iWfX6WwA2tYrbKudK+LBJ1wHjwpE7cWQO0FgwJ
 aFDahu+QD5/u45p/VcVctMtiEDvOxBdO8gfat6r+YkLm7pbRxQkZnpA/JE4Gps1p
 Td5jvMNH9pXI5pffSbeR9Q+vs/r0yqKLXQg01Eb2bZgGDgwf9yzADrHuaKamZt35
 X5flmLiTGC6swJCJvUkZC1Nuue33bXcvW5+vgvar+MNGyXsxv+B/wARLqGhiWhQZ
 nLGwFpuNu6wdY9tGHb/XR8khcewkw1/lRH1hHKhchrmRyUqHvXcPgC5tamjLrY8C
 BV3BAeQvRho8OKwWUmbXIlyON1vPux6CJdj4D/A5NL+qph2WHeVWJCXg6nVFx0Wc
 Eb3bXbI4QRwTFL7pGRF8RyZJBAQtgYhQMKWMW2GHgUgn+r1EixG73BZoSwvpHrrw
 FOR9AVNfVBqmNON8xiIb3DN4EViq76EF0jrsZh5I9EoWS2w5qtk60kJQgXE+M4EE
 vOlmh3dhEVfCN2SxOn0bgoQmTulyjqGauTSSJKQbIBuinPFveukrJfGNFIWt0SZs
 f38FBMo6sgU4VG85B+Fr
 =X5x/
 -----END PGP SIGNATURE-----

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas:
 "arm64 updates for 3.20:

   - reimplementation of the virtual remapping of UEFI Runtime Services
     in a way that is stable across kexec
   - emulation of the "setend" instruction for 32-bit tasks (user
     endianness switching trapped in the kernel, SCTLR_EL1.E0E bit set
     accordingly)
   - compat_sys_call_table implemented in C (from asm) and made it a
     constant array together with sys_call_table
   - export CPU cache information via /sys (like other architectures)
   - DMA API implementation clean-up in preparation for IOMMU support
   - macros clean-up for KVM
   - dropped some unnecessary cache+tlb maintenance
   - CONFIG_ARM64_CPU_SUSPEND clean-up
   - defconfig update (CPU_IDLE)

  The EFI changes going via the arm64 tree have been acked by Matt
  Fleming.  There is also a patch adding sys_*stat64 prototypes to
  include/linux/syscalls.h, acked by Andrew Morton"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (47 commits)
  arm64: compat: Remove incorrect comment in compat_siginfo
  arm64: Fix section mismatch on alloc_init_p[mu]d()
  arm64: Avoid breakage caused by .altmacro in fpsimd save/restore macros
  arm64: mm: use *_sect to check for section maps
  arm64: drop unnecessary cache+tlb maintenance
  arm64:mm: free the useless initial page table
  arm64: Enable CPU_IDLE in defconfig
  arm64: kernel: remove ARM64_CPU_SUSPEND config option
  arm64: make sys_call_table const
  arm64: Remove asm/syscalls.h
  arm64: Implement the compat_sys_call_table in C
  syscalls: Declare sys_*stat64 prototypes if __ARCH_WANT_(COMPAT_)STAT64
  compat: Declare compat_sys_sigpending and compat_sys_sigprocmask prototypes
  arm64: uapi: expose our struct ucontext to the uapi headers
  smp, ARM64: Kill SMP single function call interrupt
  arm64: Emulate SETEND for AArch32 tasks
  arm64: Consolidate hotplug notifier for instruction emulation
  arm64: Track system support for mixed endian EL0
  arm64: implement generic IOMMU configuration
  arm64: Combine coherent and non-coherent swiotlb dma_ops
  ...
This commit is contained in:
Linus Torvalds 2015-02-11 18:03:54 -08:00
commit 6b00f7efb5
73 changed files with 1544 additions and 930 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Documentation/arm64/legacy_instructions.txt
View File

@ -32,6 +32,9 @@ The default mode depends on the status of the instruction in the
architecture. Deprecated instructions should default to emulation
while obsolete instructions must be undefined by default.
Note: Instruction emulation may not be possible in all cases. See
individual instruction notes for further information.
Supported legacy instructions
-----------------------------
* SWP{B}
@ -43,3 +46,12 @@ Default: Undef (0)
Node: /proc/sys/abi/cp15_barrier
Status: Deprecated
Default: Emulate (1)
* SETEND
Node: /proc/sys/abi/setend
Status: Deprecated
Default: Emulate (1)*
Note: All the cpus on the system must have mixed endian support at EL0
for this feature to be enabled. If a new CPU - which doesn't support mixed
endian - is hotplugged in after this feature has been enabled, there could
be unexpected results in the application.

18
arch/arm64/Kconfig
View File

@ -540,6 +540,21 @@ config CP15_BARRIER_EMULATION
If unsure, say Y
config SETEND_EMULATION
bool "Emulate SETEND instruction"
help
The SETEND instruction alters the data-endianness of the
AArch32 EL0, and is deprecated in ARMv8.
Say Y here to enable software emulation of the instruction
for AArch32 userspace code.
Note: All the cpus on the system must have mixed endian support at EL0
for this feature to be enabled. If a new CPU - which doesn't support mixed
endian - is hotplugged in after this feature has been enabled, there could
be unexpected results in the applications.
If unsure, say Y
endif
endmenu
@ -627,9 +642,6 @@ source "kernel/power/Kconfig"
config ARCH_SUSPEND_POSSIBLE
def_bool y
config ARM64_CPU_SUSPEND
def_bool PM_SLEEP
endmenu
menu "CPU Power Management"

23
arch/arm64/Kconfig.debug
View File

@ -66,4 +66,27 @@ config DEBUG_SET_MODULE_RONX
against certain classes of kernel exploits.
If in doubt, say "N".
config DEBUG_RODATA
bool "Make kernel text and rodata read-only"
help
If this is set, kernel text and rodata will be made read-only. This
is to help catch accidental or malicious attempts to change the
kernel's executable code. Additionally splits rodata from kernel
text so it can be made explicitly non-executable.
If in doubt, say Y
config DEBUG_ALIGN_RODATA
depends on DEBUG_RODATA && !ARM64_64K_PAGES
bool "Align linker sections up to SECTION_SIZE"
help
If this option is enabled, sections that may potentially be marked as
read only or non-executable will be aligned up to the section size of
the kernel. This prevents sections from being split into pages and
avoids a potential TLB penalty. The downside is an increase in
alignment and potentially wasted space. Turn on this option if
performance is more important than memory pressure.
If in doubt, say N
endmenu

3
arch/arm64/Makefile
View File

@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
GZFLAGS :=-9
LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
KBUILD_DEFCONFIG := defconfig
KBUILD_CFLAGS += -mgeneral-regs-only
@ -50,7 +48,6 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/
core-$(CONFIG_XEN) += arch/arm64/xen/
core-$(CONFIG_CRYPTO) += arch/arm64/crypto/
libs-y := arch/arm64/lib/ $(libs-y)
libs-y += $(LIBGCC)
libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/
# Default target when executing plain make

2
arch/arm64/configs/defconfig
View File

@ -45,6 +45,8 @@ CONFIG_CMA=y
CONFIG_CMDLINE="console=ttyAMA0"
# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
CONFIG_COMPAT=y
CONFIG_CPU_IDLE=y
CONFIG_ARM64_CPUIDLE=y
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y

5
arch/arm64/include/asm/cacheflush.h
View File

@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages);
int set_memory_rw(unsigned long addr, int numpages);
int set_memory_x(unsigned long addr, int numpages);
int set_memory_nx(unsigned long addr, int numpages);
#ifdef CONFIG_DEBUG_RODATA
void mark_rodata_ro(void);
#endif
#endif

29
arch/arm64/include/asm/cachetype.h
View File

@ -39,24 +39,41 @@
extern unsigned long __icache_flags;
/*
* NumSets, bits[27:13] - (Number of sets in cache) - 1
* Associativity, bits[12:3] - (Associativity of cache) - 1
* LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2
*/
#define CCSIDR_EL1_WRITE_THROUGH BIT(31)
#define CCSIDR_EL1_WRITE_BACK BIT(30)
#define CCSIDR_EL1_READ_ALLOCATE BIT(29)
#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28)
#define CCSIDR_EL1_LINESIZE_MASK 0x7
#define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK)
#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3
#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff
#define CCSIDR_EL1_ASSOCIATIVITY(x) \
(((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK)
#define CCSIDR_EL1_NUMSETS_SHIFT 13
#define CCSIDR_EL1_NUMSETS_MASK (0x7fff << CCSIDR_EL1_NUMSETS_SHIFT)
#define CCSIDR_EL1_NUMSETS_MASK 0x7fff
#define CCSIDR_EL1_NUMSETS(x) \
(((x) & CCSIDR_EL1_NUMSETS_MASK) >> CCSIDR_EL1_NUMSETS_SHIFT)
(((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK)
extern u64 __attribute_const__ icache_get_ccsidr(void);
#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x))
#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1)
#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1)
extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr);
/* Helpers for Level 1 Instruction cache csselr = 1L */
static inline int icache_get_linesize(void)
{
return 16 << CCSIDR_EL1_LINESIZE(icache_get_ccsidr());
return CACHE_LINESIZE(cache_get_ccsidr(1L));
}
static inline int icache_get_numsets(void)
{
return 1 + CCSIDR_EL1_NUMSETS(icache_get_ccsidr());
return CACHE_NUMSETS(cache_get_ccsidr(1L));
}
/*

1
arch/arm64/include/asm/compat.h
View File

@ -161,7 +161,6 @@ typedef struct compat_siginfo {
int si_code;
union {
/* The padding is the same size as AArch64. */
int _pad[128/sizeof(int) - 3];
/* kill() */

8
arch/arm64/include/asm/cpu_ops.h
View File

@ -28,8 +28,6 @@ struct device_node;
* enable-method property.
* @cpu_init: Reads any data necessary for a specific enable-method from the
* devicetree, for a given cpu node and proposed logical id.
* @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
* devicetree, for a given cpu node and proposed logical id.
* @cpu_prepare: Early one-time preparation step for a cpu. If there is a
* mechanism for doing so, tests whether it is possible to boot
* the given CPU.
@ -42,6 +40,8 @@ struct device_node;
* @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
* cpu being killed.
* @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu.
* @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
* devicetree, for a given cpu node and proposed logical id.
* @cpu_suspend: Suspends a cpu and saves the required context. May fail owing
* to wrong parameters or error conditions. Called from the
* CPU being suspended. Must be called with IRQs disabled.
@ -49,7 +49,6 @@ struct device_node;
struct cpu_operations {
const char *name;
int (*cpu_init)(struct device_node *, unsigned int);
int (*cpu_init_idle)(struct device_node *, unsigned int);
int (*cpu_prepare)(unsigned int);
int (*cpu_boot)(unsigned int);
void (*cpu_postboot)(void);
@ -58,7 +57,8 @@ struct cpu_operations {
void (*cpu_die)(unsigned int cpu);
int (*cpu_kill)(unsigned int cpu);
#endif
#ifdef CONFIG_ARM64_CPU_SUSPEND
#ifdef CONFIG_CPU_IDLE
int (*cpu_init_idle)(struct device_node *, unsigned int);
int (*cpu_suspend)(unsigned long);
#endif
};

2
arch/arm64/include/asm/cpufeature.h
View File

@ -52,6 +52,8 @@ static inline void cpus_set_cap(unsigned int num)
}
void check_local_cpu_errata(void);
bool cpu_supports_mixed_endian_el0(void);
bool system_supports_mixed_endian_el0(void);
#endif /* __ASSEMBLY__ */

6
arch/arm64/include/asm/cpuidle.h
View File

@ -3,11 +3,17 @@
#ifdef CONFIG_CPU_IDLE
extern int cpu_init_idle(unsigned int cpu);
extern int cpu_suspend(unsigned long arg);
#else
static inline int cpu_init_idle(unsigned int cpu)
{
return -EOPNOTSUPP;
}
static inline int cpu_suspend(unsigned long arg)
{
return -EOPNOTSUPP;
}
#endif
#endif

17
arch/arm64/include/asm/cputype.h
View File

@ -72,6 +72,18 @@
#define APM_CPU_PART_POTENZA 0x000
#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16
#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)
#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \
(((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT)
#define ID_AA64MMFR0_BIGEND_SHIFT 8
#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT)
#define ID_AA64MMFR0_BIGEND(mmfr0) \
(((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
#define SCTLR_EL1_CP15BEN (0x1 << 5)
#define SCTLR_EL1_SED (0x1 << 8)
#ifndef __ASSEMBLY__
/*
@ -104,6 +116,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void)
return read_cpuid(CTR_EL0);
}
static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
{
return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) ||
(ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1);
}
#endif /* __ASSEMBLY__ */
#endif

11
arch/arm64/include/asm/dma-mapping.h
View File

@ -28,8 +28,6 @@
#define DMA_ERROR_CODE (~(dma_addr_t)0)
extern struct dma_map_ops *dma_ops;
extern struct dma_map_ops coherent_swiotlb_dma_ops;
extern struct dma_map_ops noncoherent_swiotlb_dma_ops;
static inline struct dma_map_ops *__generic_dma_ops(struct device *dev)
{
@ -47,23 +45,18 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
return __generic_dma_ops(dev);
}
static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops)
{
dev->archdata.dma_ops = ops;
}
static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
struct iommu_ops *iommu, bool coherent)
{
dev->archdata.dma_coherent = coherent;
if (coherent)
set_dma_ops(dev, &coherent_swiotlb_dma_ops);
}
#define arch_setup_dma_ops arch_setup_dma_ops
/* do not use this function in a driver */
static inline bool is_device_dma_coherent(struct device *dev)
{
if (!dev)
return false;
return dev->archdata.dma_coherent;
}

30
arch/arm64/include/asm/efi.h
View File

@ -6,29 +6,33 @@
#ifdef CONFIG_EFI
extern void efi_init(void);
extern void efi_idmap_init(void);
#else
#define efi_init()
#define efi_idmap_init()
#endif
#define efi_call_virt(f, ...) \
({ \
efi_##f##_t *__f = efi.systab->runtime->f; \
efi_##f##_t *__f; \
efi_status_t __s; \
\
kernel_neon_begin(); \
efi_virtmap_load(); \
__f = efi.systab->runtime->f; \
__s = __f(__VA_ARGS__); \
efi_virtmap_unload(); \
kernel_neon_end(); \
__s; \
})
#define __efi_call_virt(f, ...) \
({ \
efi_##f##_t *__f = efi.systab->runtime->f; \
efi_##f##_t *__f; \
\
kernel_neon_begin(); \
efi_virtmap_load(); \
__f = efi.systab->runtime->f; \
__f(__VA_ARGS__); \
efi_virtmap_unload(); \
kernel_neon_end(); \
})
@ -44,4 +48,22 @@ extern void efi_idmap_init(void);
#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__)
#define EFI_ALLOC_ALIGN SZ_64K
/*
* On ARM systems, virtually remapped UEFI runtime services are set up in two
* distinct stages:
* - The stub retrieves the final version of the memory map from UEFI, populates
* the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime
* service to communicate the new mapping to the firmware (Note that the new
* mapping is not live at this time)
* - During an early initcall(), the EFI system table is permanently remapped
* and the virtual remapping of the UEFI Runtime Services regions is loaded
* into a private set of page tables. If this all succeeds, the Runtime
* Services are enabled and the EFI_RUNTIME_SERVICES bit set.
*/
void efi_virtmap_load(void);
void efi_virtmap_unload(void);
#endif /* _ASM_EFI_H */

117
arch/arm64/include/asm/esr.h
View File

@ -18,40 +18,89 @@
#ifndef __ASM_ESR_H
#define __ASM_ESR_H
#define ESR_EL1_WRITE (1 << 6)
#define ESR_EL1_CM (1 << 8)
#define ESR_EL1_IL (1 << 25)
#define ESR_ELx_EC_UNKNOWN (0x00)
#define ESR_ELx_EC_WFx (0x01)
/* Unallocated EC: 0x02 */
#define ESR_ELx_EC_CP15_32 (0x03)
#define ESR_ELx_EC_CP15_64 (0x04)
#define ESR_ELx_EC_CP14_MR (0x05)
#define ESR_ELx_EC_CP14_LS (0x06)
#define ESR_ELx_EC_FP_ASIMD (0x07)
#define ESR_ELx_EC_CP10_ID (0x08)
/* Unallocated EC: 0x09 - 0x0B */
#define ESR_ELx_EC_CP14_64 (0x0C)
/* Unallocated EC: 0x0d */
#define ESR_ELx_EC_ILL (0x0E)
/* Unallocated EC: 0x0F - 0x10 */
#define ESR_ELx_EC_SVC32 (0x11)
#define ESR_ELx_EC_HVC32 (0x12)
#define ESR_ELx_EC_SMC32 (0x13)
/* Unallocated EC: 0x14 */
#define ESR_ELx_EC_SVC64 (0x15)
#define ESR_ELx_EC_HVC64 (0x16)
#define ESR_ELx_EC_SMC64 (0x17)
#define ESR_ELx_EC_SYS64 (0x18)
/* Unallocated EC: 0x19 - 0x1E */
#define ESR_ELx_EC_IMP_DEF (0x1f)
#define ESR_ELx_EC_IABT_LOW (0x20)
#define ESR_ELx_EC_IABT_CUR (0x21)
#define ESR_ELx_EC_PC_ALIGN (0x22)
/* Unallocated EC: 0x23 */
#define ESR_ELx_EC_DABT_LOW (0x24)
#define ESR_ELx_EC_DABT_CUR (0x25)
#define ESR_ELx_EC_SP_ALIGN (0x26)
/* Unallocated EC: 0x27 */
#define ESR_ELx_EC_FP_EXC32 (0x28)
/* Unallocated EC: 0x29 - 0x2B */
#define ESR_ELx_EC_FP_EXC64 (0x2C)
/* Unallocated EC: 0x2D - 0x2E */
#define ESR_ELx_EC_SERROR (0x2F)
#define ESR_ELx_EC_BREAKPT_LOW (0x30)
#define ESR_ELx_EC_BREAKPT_CUR (0x31)
#define ESR_ELx_EC_SOFTSTP_LOW (0x32)
#define ESR_ELx_EC_SOFTSTP_CUR (0x33)
#define ESR_ELx_EC_WATCHPT_LOW (0x34)
#define ESR_ELx_EC_WATCHPT_CUR (0x35)
/* Unallocated EC: 0x36 - 0x37 */
#define ESR_ELx_EC_BKPT32 (0x38)
/* Unallocated EC: 0x39 */
#define ESR_ELx_EC_VECTOR32 (0x3A)
/* Unallocted EC: 0x3B */
#define ESR_ELx_EC_BRK64 (0x3C)
/* Unallocated EC: 0x3D - 0x3F */
#define ESR_ELx_EC_MAX (0x3F)
#define ESR_EL1_EC_SHIFT (26)
#define ESR_EL1_EC_UNKNOWN (0x00)
#define ESR_EL1_EC_WFI (0x01)
#define ESR_EL1_EC_CP15_32 (0x03)
#define ESR_EL1_EC_CP15_64 (0x04)
#define ESR_EL1_EC_CP14_MR (0x05)
#define ESR_EL1_EC_CP14_LS (0x06)
#define ESR_EL1_EC_FP_ASIMD (0x07)
#define ESR_EL1_EC_CP10_ID (0x08)
#define ESR_EL1_EC_CP14_64 (0x0C)
#define ESR_EL1_EC_ILL_ISS (0x0E)
#define ESR_EL1_EC_SVC32 (0x11)
#define ESR_EL1_EC_SVC64 (0x15)
#define ESR_EL1_EC_SYS64 (0x18)
#define ESR_EL1_EC_IABT_EL0 (0x20)
#define ESR_EL1_EC_IABT_EL1 (0x21)
#define ESR_EL1_EC_PC_ALIGN (0x22)
#define ESR_EL1_EC_DABT_EL0 (0x24)
#define ESR_EL1_EC_DABT_EL1 (0x25)
#define ESR_EL1_EC_SP_ALIGN (0x26)
#define ESR_EL1_EC_FP_EXC32 (0x28)
#define ESR_EL1_EC_FP_EXC64 (0x2C)
#define ESR_EL1_EC_SERROR (0x2F)
#define ESR_EL1_EC_BREAKPT_EL0 (0x30)
#define ESR_EL1_EC_BREAKPT_EL1 (0x31)
#define ESR_EL1_EC_SOFTSTP_EL0 (0x32)
#define ESR_EL1_EC_SOFTSTP_EL1 (0x33)
#define ESR_EL1_EC_WATCHPT_EL0 (0x34)
#define ESR_EL1_EC_WATCHPT_EL1 (0x35)
#define ESR_EL1_EC_BKPT32 (0x38)
#define ESR_EL1_EC_BRK64 (0x3C)
#define ESR_ELx_EC_SHIFT (26)
#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
#define ESR_ELx_IL (UL(1) << 25)
#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)
#define ESR_ELx_ISV (UL(1) << 24)
#define ESR_ELx_SAS_SHIFT (22)
#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)
#define ESR_ELx_SSE (UL(1) << 21)
#define ESR_ELx_SRT_SHIFT (16)
#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)
#define ESR_ELx_SF (UL(1) << 15)
#define ESR_ELx_AR (UL(1) << 14)
#define ESR_ELx_EA (UL(1) << 9)
#define ESR_ELx_CM (UL(1) << 8)
#define ESR_ELx_S1PTW (UL(1) << 7)
#define ESR_ELx_WNR (UL(1) << 6)
#define ESR_ELx_FSC (0x3F)
#define ESR_ELx_FSC_TYPE (0x3C)
#define ESR_ELx_FSC_EXTABT (0x10)
#define ESR_ELx_FSC_FAULT (0x04)
#define ESR_ELx_FSC_PERM (0x0C)
#define ESR_ELx_CV (UL(1) << 24)
#define ESR_ELx_COND_SHIFT (20)
#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT)
#define ESR_ELx_WFx_ISS_WFE (UL(1) << 0)
#ifndef __ASSEMBLY__
#include <asm/types.h>
const char *esr_get_class_string(u32 esr);
#endif /* __ASSEMBLY */
#endif /* __ASM_ESR_H */

1
arch/arm64/include/asm/fixmap.h
View File

@ -49,6 +49,7 @@ enum fixed_addresses {
FIX_BTMAP_END = __end_of_permanent_fixed_addresses,
FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1,
FIX_TEXT_POKE0,
__end_of_fixed_addresses
};

43
arch/arm64/include/asm/fpsimdmacros.h
View File

@ -76,7 +76,6 @@
fpsimd_restore_fpcr x\tmpnr, \state
.endm
.altmacro
.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
mrs x\tmpnr1, fpsr
str w\numnr, [\state, #8]
@ -86,11 +85,22 @@
add \state, \state, x\numnr, lsl #4
sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
br x\tmpnr1
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0
.irp qb, %(qa + 1)
stp q\qa, q\qb, [\state, # -16 * \qa - 16]
.endr
.endr
stp q30, q31, [\state, #-16 * 30 - 16]
stp q28, q29, [\state, #-16 * 28 - 16]
stp q26, q27, [\state, #-16 * 26 - 16]
stp q24, q25, [\state, #-16 * 24 - 16]
stp q22, q23, [\state, #-16 * 22 - 16]
stp q20, q21, [\state, #-16 * 20 - 16]
stp q18, q19, [\state, #-16 * 18 - 16]
stp q16, q17, [\state, #-16 * 16 - 16]
stp q14, q15, [\state, #-16 * 14 - 16]
stp q12, q13, [\state, #-16 * 12 - 16]
stp q10, q11, [\state, #-16 * 10 - 16]
stp q8, q9, [\state, #-16 * 8 - 16]
stp q6, q7, [\state, #-16 * 6 - 16]
stp q4, q5, [\state, #-16 * 4 - 16]
stp q2, q3, [\state, #-16 * 2 - 16]
stp q0, q1, [\state, #-16 * 0 - 16]
0:
.endm
@ -103,10 +113,21 @@
add \state, \state, x\tmpnr2, lsl #4
sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
br x\tmpnr1
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0
.irp qb, %(qa + 1)
ldp q\qa, q\qb, [\state, # -16 * \qa - 16]
.endr
.endr
ldp q30, q31, [\state, #-16 * 30 - 16]
ldp q28, q29, [\state, #-16 * 28 - 16]
ldp q26, q27, [\state, #-16 * 26 - 16]
ldp q24, q25, [\state, #-16 * 24 - 16]
ldp q22, q23, [\state, #-16 * 22 - 16]
ldp q20, q21, [\state, #-16 * 20 - 16]
ldp q18, q19, [\state, #-16 * 18 - 16]
ldp q16, q17, [\state, #-16 * 16 - 16]
ldp q14, q15, [\state, #-16 * 14 - 16]
ldp q12, q13, [\state, #-16 * 12 - 16]
ldp q10, q11, [\state, #-16 * 10 - 16]
ldp q8, q9, [\state, #-16 * 8 - 16]
ldp q6, q7, [\state, #-16 * 6 - 16]
ldp q4, q5, [\state, #-16 * 4 - 16]
ldp q2, q3, [\state, #-16 * 2 - 16]
ldp q0, q1, [\state, #-16 * 0 - 16]
0:
.endm

2
arch/arm64/include/asm/hardirq.h
View File

@ -20,7 +20,7 @@
#include <linux/threads.h>
#include <asm/irq.h>
#define NR_IPI 6
#define NR_IPI 5
typedef struct {
unsigned int __softirq_pending;

5
arch/arm64/include/asm/io.h
View File

@ -26,6 +26,7 @@
#include <asm/byteorder.h>
#include <asm/barrier.h>
#include <asm/memory.h>
#include <asm/pgtable.h>
#include <asm/early_ioremap.h>
#include <asm/alternative.h>
@ -145,8 +146,8 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
* I/O port access primitives.
*/
#define arch_has_dev_port() (1)
#define IO_SPACE_LIMIT (SZ_32M - 1)
#define PCI_IOBASE ((void __iomem *)(MODULES_VADDR - SZ_32M))
#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1)
#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
/*
* String version of I/O memory access operations.

73
arch/arm64/include/asm/kvm_arm.h
View File

@ -18,6 +18,7 @@
#ifndef __ARM64_KVM_ARM_H__
#define __ARM64_KVM_ARM_H__
#include <asm/esr.h>
#include <asm/memory.h>
#include <asm/types.h>
@ -184,77 +185,11 @@
#define MDCR_EL2_TPMCR (1 << 5)
#define MDCR_EL2_HPMN_MASK (0x1F)
/* Exception Syndrome Register (ESR) bits */
#define ESR_EL2_EC_SHIFT (26)
#define ESR_EL2_EC (UL(0x3f) << ESR_EL2_EC_SHIFT)
#define ESR_EL2_IL (UL(1) << 25)
#define ESR_EL2_ISS (ESR_EL2_IL - 1)
#define ESR_EL2_ISV_SHIFT (24)
#define ESR_EL2_ISV (UL(1) << ESR_EL2_ISV_SHIFT)
#define ESR_EL2_SAS_SHIFT (22)
#define ESR_EL2_SAS (UL(3) << ESR_EL2_SAS_SHIFT)
#define ESR_EL2_SSE (1 << 21)
#define ESR_EL2_SRT_SHIFT (16)
#define ESR_EL2_SRT_MASK (0x1f << ESR_EL2_SRT_SHIFT)
#define ESR_EL2_SF (1 << 15)
#define ESR_EL2_AR (1 << 14)
#define ESR_EL2_EA (1 << 9)
#define ESR_EL2_CM (1 << 8)
#define ESR_EL2_S1PTW (1 << 7)
#define ESR_EL2_WNR (1 << 6)
#define ESR_EL2_FSC (0x3f)
#define ESR_EL2_FSC_TYPE (0x3c)
#define ESR_EL2_CV_SHIFT (24)
#define ESR_EL2_CV (UL(1) << ESR_EL2_CV_SHIFT)
#define ESR_EL2_COND_SHIFT (20)
#define ESR_EL2_COND (UL(0xf) << ESR_EL2_COND_SHIFT)
#define FSC_FAULT (0x04)
#define FSC_PERM (0x0c)
/* For compatibility with fault code shared with 32-bit */
#define FSC_FAULT ESR_ELx_FSC_FAULT
#define FSC_PERM ESR_ELx_FSC_PERM
/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
#define HPFAR_MASK (~UL(0xf))
#define ESR_EL2_EC_UNKNOWN (0x00)
#define ESR_EL2_EC_WFI (0x01)
#define ESR_EL2_EC_CP15_32 (0x03)
#define ESR_EL2_EC_CP15_64 (0x04)
#define ESR_EL2_EC_CP14_MR (0x05)
#define ESR_EL2_EC_CP14_LS (0x06)
#define ESR_EL2_EC_FP_ASIMD (0x07)
#define ESR_EL2_EC_CP10_ID (0x08)
#define ESR_EL2_EC_CP14_64 (0x0C)
#define ESR_EL2_EC_ILL_ISS (0x0E)
#define ESR_EL2_EC_SVC32 (0x11)
#define ESR_EL2_EC_HVC32 (0x12)
#define ESR_EL2_EC_SMC32 (0x13)
#define ESR_EL2_EC_SVC64 (0x15)
#define ESR_EL2_EC_HVC64 (0x16)
#define ESR_EL2_EC_SMC64 (0x17)
#define ESR_EL2_EC_SYS64 (0x18)
#define ESR_EL2_EC_IABT (0x20)
#define ESR_EL2_EC_IABT_HYP (0x21)
#define ESR_EL2_EC_PC_ALIGN (0x22)
#define ESR_EL2_EC_DABT (0x24)
#define ESR_EL2_EC_DABT_HYP (0x25)
#define ESR_EL2_EC_SP_ALIGN (0x26)
#define ESR_EL2_EC_FP_EXC32 (0x28)
#define ESR_EL2_EC_FP_EXC64 (0x2C)
#define ESR_EL2_EC_SERROR (0x2F)
#define ESR_EL2_EC_BREAKPT (0x30)
#define ESR_EL2_EC_BREAKPT_HYP (0x31)
#define ESR_EL2_EC_SOFTSTP (0x32)
#define ESR_EL2_EC_SOFTSTP_HYP (0x33)
#define ESR_EL2_EC_WATCHPT (0x34)
#define ESR_EL2_EC_WATCHPT_HYP (0x35)
#define ESR_EL2_EC_BKPT32 (0x38)
#define ESR_EL2_EC_VECTOR32 (0x3A)
#define ESR_EL2_EC_BRK64 (0x3C)
#define ESR_EL2_EC_xABT_xFSR_EXTABT 0x10
#define ESR_EL2_EC_WFI_ISS_WFE (1 << 0)
#endif /* __ARM64_KVM_ARM_H__ */

28
arch/arm64/include/asm/kvm_emulate.h
View File

@ -23,8 +23,10 @@
#define __ARM64_KVM_EMULATE_H__
#include <linux/kvm_host.h>
#include <asm/kvm_asm.h>
#include <asm/esr.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmio.h>
#include <asm/ptrace.h>
@ -140,63 +142,63 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_ISV);
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
}
static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_WNR);
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR);
}
static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SSE);
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE);
}
static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
{
return (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SRT_MASK) >> ESR_EL2_SRT_SHIFT;
return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
}
static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EA);
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_EA);
}
static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_S1PTW);
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW);
}
static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
{
return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SAS) >> ESR_EL2_SAS_SHIFT);
return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
}
/* This one is not specific to Data Abort */
static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_IL);
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL);
}
static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_hsr(vcpu) >> ESR_EL2_EC_SHIFT;
return kvm_vcpu_get_hsr(vcpu) >> ESR_ELx_EC_SHIFT;
}
static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_trap_get_class(vcpu) == ESR_EL2_EC_IABT;
return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
}
static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC;
return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC;
}
static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC_TYPE;
return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
}
static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)

10
arch/arm64/include/asm/memory.h
View File

@ -32,6 +32,12 @@
*/
#define UL(x) _AC(x, UL)
/*
* Size of the PCI I/O space. This must remain a power of two so that
* IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses.
*/
#define PCI_IO_SIZE SZ_16M
/*
* PAGE_OFFSET - the virtual address of the start of the kernel image (top
* (VA_BITS - 1))
@ -45,7 +51,9 @@
#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
#define MODULES_END (PAGE_OFFSET)
#define MODULES_VADDR (MODULES_END - SZ_64M)
#define FIXADDR_TOP (MODULES_VADDR - SZ_2M - PAGE_SIZE)
#define PCI_IO_END (MODULES_VADDR - SZ_2M)
#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
#define TASK_SIZE_64 (UL(1) << VA_BITS)
#ifdef CONFIG_COMPAT

5
arch/arm64/include/asm/mmu.h
View File

@ -31,7 +31,8 @@ extern void paging_init(void);
extern void setup_mm_for_reboot(void);
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
extern void init_mem_pgprot(void);
/* create an identity mapping for memory (or io if map_io is true) */
extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io);
extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot);
#endif

8
arch/arm64/include/asm/pgtable.h
View File

@ -263,6 +263,11 @@ static inline pmd_t pte_pmd(pte_t pte)
return __pmd(pte_val(pte));
}
static inline pgprot_t mk_sect_prot(pgprot_t prot)
{
return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
}
/*
* THP definitions.
*/
@ -336,9 +341,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
#ifdef CONFIG_ARM64_64K_PAGES
#define pud_sect(pud) (0)
#define pud_table(pud) (1)
#else
#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
PUD_TYPE_SECT)
#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
PUD_TYPE_TABLE)
#endif
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)

7
arch/arm64/include/asm/ptrace.h
View File

@ -58,6 +58,13 @@
#define COMPAT_PSR_Z_BIT 0x40000000
#define COMPAT_PSR_N_BIT 0x80000000
#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
#ifdef CONFIG_CPU_BIG_ENDIAN
#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT
#else
#define COMPAT_PSR_ENDSTATE 0
#endif
/*
* These are 'magic' values for PTRACE_PEEKUSR that return info about where a
* process is located in memory.

2
arch/arm64/include/asm/suspend.h
View File

@ -23,6 +23,4 @@ struct sleep_save_sp {
extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
extern void cpu_resume(void);
extern int cpu_suspend(unsigned long);
#endif

30
arch/arm64/include/asm/syscalls.h
View File

@ -1,30 +0,0 @@
/*
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_SYSCALLS_H
#define __ASM_SYSCALLS_H
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/signal.h>
/*
* System call wrappers implemented in kernel/entry.S.
*/
asmlinkage long sys_rt_sigreturn_wrapper(void);
#include <asm-generic/syscalls.h>
#endif /* __ASM_SYSCALLS_H */

3
arch/arm64/include/asm/unistd.h
View File

@ -48,6 +48,9 @@
#endif
#define __ARCH_WANT_SYS_CLONE
#ifndef __COMPAT_SYSCALL_NR
#include <uapi/asm/unistd.h>
#endif
#define NR_syscalls (__NR_syscalls)

1
arch/arm64/include/uapi/asm/Kbuild
View File

@ -18,4 +18,5 @@ header-y += siginfo.h
header-y += signal.h
header-y += stat.h
header-y += statfs.h
header-y += ucontext.h
header-y += unistd.h

8
arch/arm64/include/asm/ucontext.h → arch/arm64/include/uapi/asm/ucontext.h
View File

@ -13,8 +13,10 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_UCONTEXT_H
#define __ASM_UCONTEXT_H
#ifndef _UAPI__ASM_UCONTEXT_H
#define _UAPI__ASM_UCONTEXT_H
#include <linux/types.h>
struct ucontext {
unsigned long uc_flags;
@ -27,4 +29,4 @@ struct ucontext {
struct sigcontext uc_mcontext;
};
#endif /* __ASM_UCONTEXT_H */
#endif /* _UAPI__ASM_UCONTEXT_H */

6
arch/arm64/kernel/Makefile
View File

@ -16,10 +16,10 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
entry-fpsimd.o process.o ptrace.o setup.o signal.o \
sys.o stacktrace.o time.o traps.o io.o vdso.o \
hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \
cpuinfo.o cpu_errata.o alternative.o
cpuinfo.o cpu_errata.o alternative.o cacheinfo.o
arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
sys_compat.o \
sys_compat.o entry32.o \
../../arm/kernel/opcodes.o
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
@ -27,7 +27,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o
arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o
arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o
arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o
arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
arm64-obj-$(CONFIG_KGDB) += kgdb.o

205
arch/arm64/kernel/armv8_deprecated.c
View File

@ -19,6 +19,7 @@
#include <asm/system_misc.h>
#include <asm/traps.h>
#include <asm/uaccess.h>
#include <asm/cpufeature.h>
#define CREATE_TRACE_POINTS
#include "trace-events-emulation.h"
@ -85,6 +86,57 @@ static void remove_emulation_hooks(struct insn_emulation_ops *ops)
pr_notice("Removed %s emulation handler\n", ops->name);
}
static void enable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->ops->set_hw_mode)
insn->ops->set_hw_mode(true);
}
static void disable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->ops->set_hw_mode)
insn->ops->set_hw_mode(false);
}
/* Run set_hw_mode(mode) on all active CPUs */
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
{
if (!insn->ops->set_hw_mode)
return -EINVAL;
if (enable)
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
else
on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
return 0;
}
/*
* Run set_hw_mode for all insns on a starting CPU.
* Returns:
* 0 - If all the hooks ran successfully.
* -EINVAL - At least one hook is not supported by the CPU.
*/
static int run_all_insn_set_hw_mode(unsigned long cpu)
{
int rc = 0;
unsigned long flags;
struct insn_emulation *insn;
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
list_for_each_entry(insn, &insn_emulation, node) {
bool enable = (insn->current_mode == INSN_HW);
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
pr_warn("CPU[%ld] cannot support the emulation of %s",
cpu, insn->ops->name);
rc = -EINVAL;
}
}
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
return rc;
}
static int update_insn_emulation_mode(struct insn_emulation *insn,
enum insn_emulation_mode prev)
{
@ -97,10 +149,8 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
remove_emulation_hooks(insn->ops);
break;
case INSN_HW:
if (insn->ops->set_hw_mode) {
insn->ops->set_hw_mode(false);
if (!run_all_cpu_set_hw_mode(insn, false))
pr_notice("Disabled %s support\n", insn->ops->name);
}
break;
}
@ -111,10 +161,9 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
register_emulation_hooks(insn->ops);
break;
case INSN_HW:
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true))
ret = run_all_cpu_set_hw_mode(insn, true);
if (!ret)
pr_notice("Enabled %s support\n", insn->ops->name);
else
ret = -EINVAL;
break;
}
@ -133,6 +182,8 @@ static void register_insn_emulation(struct insn_emulation_ops *ops)
switch (ops->status) {
case INSN_DEPRECATED:
insn->current_mode = INSN_EMULATE;
/* Disable the HW mode if it was turned on at early boot time */
run_all_cpu_set_hw_mode(insn, false);
insn->max = INSN_HW;
break;
case INSN_OBSOLETE:
@ -453,8 +504,6 @@ ret:
return 0;
}
#define SCTLR_EL1_CP15BEN (1 << 5)
static inline void config_sctlr_el1(u32 clear, u32 set)
{
u32 val;
@ -465,48 +514,13 @@ static inline void config_sctlr_el1(u32 clear, u32 set)
asm volatile("msr sctlr_el1, %0" : : "r" (val));
}
static void enable_cp15_ben(void *info)
{
config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
}
static void disable_cp15_ben(void *info)
{
config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
}
static int cpu_hotplug_notify(struct notifier_block *b,
unsigned long action, void *hcpu)
{
switch (action) {
case CPU_STARTING:
case CPU_STARTING_FROZEN:
enable_cp15_ben(NULL);
return NOTIFY_DONE;
case CPU_DYING:
case CPU_DYING_FROZEN:
disable_cp15_ben(NULL);
return NOTIFY_DONE;
}
return NOTIFY_OK;
}
static struct notifier_block cpu_hotplug_notifier = {
.notifier_call = cpu_hotplug_notify,
};
static int cp15_barrier_set_hw_mode(bool enable)
{
if (enable) {
register_cpu_notifier(&cpu_hotplug_notifier);
on_each_cpu(enable_cp15_ben, NULL, true);
} else {
unregister_cpu_notifier(&cpu_hotplug_notifier);
on_each_cpu(disable_cp15_ben, NULL, true);
}
return true;
if (enable)
config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
else
config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
return 0;
}
static struct undef_hook cp15_barrier_hooks[] = {
@ -534,6 +548,93 @@ static struct insn_emulation_ops cp15_barrier_ops = {
.set_hw_mode = cp15_barrier_set_hw_mode,
};
static int setend_set_hw_mode(bool enable)
{
if (!cpu_supports_mixed_endian_el0())
return -EINVAL;
if (enable)
config_sctlr_el1(SCTLR_EL1_SED, 0);
else
config_sctlr_el1(0, SCTLR_EL1_SED);
return 0;
}
static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
{
char *insn;
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
if (big_endian) {
insn = "setend be";
regs->pstate |= COMPAT_PSR_E_BIT;
} else {
insn = "setend le";
regs->pstate &= ~COMPAT_PSR_E_BIT;
}
trace_instruction_emulation(insn, regs->pc);
pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
current->comm, (unsigned long)current->pid, regs->pc);
return 0;
}
static int a32_setend_handler(struct pt_regs *regs, u32 instr)
{
int rc = compat_setend_handler(regs, (instr >> 9) & 1);
regs->pc += 4;
return rc;
}
static int t16_setend_handler(struct pt_regs *regs, u32 instr)
{
int rc = compat_setend_handler(regs, (instr >> 3) & 1);
regs->pc += 2;
return rc;
}
static struct undef_hook setend_hooks[] = {
{
.instr_mask = 0xfffffdff,
.instr_val = 0xf1010000,
.pstate_mask = COMPAT_PSR_MODE_MASK,
.pstate_val = COMPAT_PSR_MODE_USR,
.fn = a32_setend_handler,
},
{
/* Thumb mode */
.instr_mask = 0x0000fff7,
.instr_val = 0x0000b650,
.pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK),
.pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR),
.fn = t16_setend_handler,
},
{}
};
static struct insn_emulation_ops setend_ops = {
.name = "setend",
.status = INSN_DEPRECATED,
.hooks = setend_hooks,
.set_hw_mode = setend_set_hw_mode,
};
static int insn_cpu_hotplug_notify(struct notifier_block *b,
unsigned long action, void *hcpu)
{
int rc = 0;
if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING)
rc = run_all_insn_set_hw_mode((unsigned long)hcpu);
return notifier_from_errno(rc);
}
static struct notifier_block insn_cpu_hotplug_notifier = {
.notifier_call = insn_cpu_hotplug_notify,
};
/*
* Invoked as late_initcall, since not needed before init spawned.
*/
@ -545,6 +646,14 @@ static int __init armv8_deprecated_init(void)
if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
register_insn_emulation(&cp15_barrier_ops);
if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
if(system_supports_mixed_endian_el0())
register_insn_emulation(&setend_ops);
else
pr_info("setend instruction emulation is not supported on the system");
}
register_cpu_notifier(&insn_cpu_hotplug_notifier);
register_insn_emulation_sysctl(ctl_abi);
return 0;

2
arch/arm64/kernel/asm-offsets.c
View File

@ -152,7 +152,7 @@ int main(void)
DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
#endif
#ifdef CONFIG_ARM64_CPU_SUSPEND
#ifdef CONFIG_CPU_PM
DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));

128
arch/arm64/kernel/cacheinfo.c Normal file
View File

@ -0,0 +1,128 @@
/*
* ARM64 cacheinfo support
*
* Copyright (C) 2015 ARM Ltd.
* All Rights Reserved
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed "as is" WITHOUT ANY WARRANTY of any
* kind, whether express or implied; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/bitops.h>
#include <linux/cacheinfo.h>
#include <linux/cpu.h>
#include <linux/compiler.h>
#include <linux/of.h>
#include <asm/cachetype.h>
#include <asm/processor.h>
#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */
/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */
#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1))
#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level))
#define CLIDR_CTYPE(clidr, level) \
(((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level))
static inline enum cache_type get_cache_type(int level)
{
u64 clidr;
if (level > MAX_CACHE_LEVEL)
return CACHE_TYPE_NOCACHE;
asm volatile ("mrs %x0, clidr_el1" : "=r" (clidr));
return CLIDR_CTYPE(clidr, level);
}
/*
* Cache Size Selection Register(CSSELR) selects which Cache Size ID
* Register(CCSIDR) is accessible by specifying the required cache
* level and the cache type. We need to ensure that no one else changes
* CSSELR by calling this in non-preemtible context
*/
u64 __attribute_const__ cache_get_ccsidr(u64 csselr)
{
u64 ccsidr;
WARN_ON(preemptible());
/* Put value into CSSELR */
asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
isb();
/* Read result out of CCSIDR */
asm volatile("mrs %x0, ccsidr_el1" : "=r" (ccsidr));
return ccsidr;
}
static void ci_leaf_init(struct cacheinfo *this_leaf,
enum cache_type type, unsigned int level)
{
bool is_icache = type & CACHE_TYPE_INST;
u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache);
this_leaf->level = level;
this_leaf->type = type;
this_leaf->coherency_line_size = CACHE_LINESIZE(tmp);
this_leaf->number_of_sets = CACHE_NUMSETS(tmp);
this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp);
this_leaf->size = this_leaf->number_of_sets *
this_leaf->coherency_line_size * this_leaf->ways_of_associativity;
this_leaf->attributes =
((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) |
((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) |
((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) |
((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0);
}
static int __init_cache_level(unsigned int cpu)
{
unsigned int ctype, level, leaves;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) {
ctype = get_cache_type(level);
if (ctype == CACHE_TYPE_NOCACHE) {
level--;
break;
}
/* Separate instruction and data caches */
leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1;
}
this_cpu_ci->num_levels = level;
this_cpu_ci->num_leaves = leaves;
return 0;
}
static int __populate_cache_leaves(unsigned int cpu)
{
unsigned int level, idx;
enum cache_type type;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
struct cacheinfo *this_leaf = this_cpu_ci->info_list;
for (idx = 0, level = 1; level <= this_cpu_ci->num_levels &&
idx < this_cpu_ci->num_leaves; idx++, level++) {
type = get_cache_type(level);
if (type == CACHE_TYPE_SEPARATE) {
ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level);
ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level);
} else {
ci_leaf_init(this_leaf++, type, level);
}
}
return 0;
}
DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level)
DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves)

20
arch/arm64/kernel/cpuidle.c
View File

@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu)
of_node_put(cpu_node);
return ret;
}
/**
* cpu_suspend() - function to enter a low-power idle state
* @arg: argument to pass to CPU suspend operations
*
* Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
* operations back-end error code otherwise.
*/
int cpu_suspend(unsigned long arg)
{
int cpu = smp_processor_id();
/*
* If cpu_ops have not been registered or suspend
* has not been initialized, cpu_suspend call fails early.
*/
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
return -EOPNOTSUPP;
return cpu_ops[cpu]->cpu_suspend(arg);
}

34
arch/arm64/kernel/cpuinfo.c
View File

@ -35,6 +35,7 @@
*/
DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
static struct cpuinfo_arm64 boot_cpu_data;
static bool mixed_endian_el0 = true;
static char *icache_policy_str[] = {
[ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN",
@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
}
bool cpu_supports_mixed_endian_el0(void)
{
return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
}
bool system_supports_mixed_endian_el0(void)
{
return mixed_endian_el0;
}
static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info)
{
mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0);
}
static void update_cpu_features(struct cpuinfo_arm64 *info)
{
update_mixed_endian_el0_support(info);
}
static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu)
{
if ((boot & mask) == (cur & mask))
@ -215,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
cpuinfo_detect_icache_policy(info);
check_local_cpu_errata();
update_cpu_features(info);
}
void cpuinfo_store_cpu(void)
@ -231,15 +253,3 @@ void __init cpuinfo_store_boot_cpu(void)
boot_cpu_data = *info;
}
u64 __attribute_const__ icache_get_ccsidr(void)
{
u64 ccsidr;
WARN_ON(preemptible());
/* Select L1 I-cache and read its size ID register */
asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1"
: "=r"(ccsidr) : "r"(1L));
return ccsidr;
}

354
arch/arm64/kernel/efi.c
View File

@ -11,27 +11,46 @@
*
*/
#include <linux/atomic.h>
#include <linux/dmi.h>
#include <linux/efi.h>
#include <linux/export.h>
#include <linux/memblock.h>
#include <linux/mm_types.h>
#include <linux/bootmem.h>
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/preempt.h>
#include <linux/rbtree.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <asm/cacheflush.h>
#include <asm/efi.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/mmu.h>
#include <asm/pgtable.h>
struct efi_memory_map memmap;
static efi_runtime_services_t *runtime;
static u64 efi_system_table;
static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss;
static struct mm_struct efi_mm = {
.mm_rb = RB_ROOT,
.pgd = efi_pgd,
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),
.mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem),
.page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
.mmlist = LIST_HEAD_INIT(efi_mm.mmlist),
INIT_MM_CONTEXT(efi_mm)
};
static int uefi_debug __initdata;
static int __init uefi_debug_setup(char *str)
{
@ -48,30 +67,33 @@ static int __init is_normal_ram(efi_memory_desc_t *md)
return 0;
}
static void __init efi_setup_idmap(void)
/*
* Translate a EFI virtual address into a physical address: this is necessary,
* as some data members of the EFI system table are virtually remapped after
* SetVirtualAddressMap() has been called.
*/
static phys_addr_t efi_to_phys(unsigned long addr)
{
struct memblock_region *r;
efi_memory_desc_t *md;
u64 paddr, npages, size;
for_each_memblock(memory, r)
create_id_mapping(r->base, r->size, 0);
/* map runtime io spaces */
for_each_efi_memory_desc(&memmap, md) {
if (!(md->attribute & EFI_MEMORY_RUNTIME) || is_normal_ram(md))
if (!(md->attribute & EFI_MEMORY_RUNTIME))
continue;
paddr = md->phys_addr;
npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
create_id_mapping(paddr, size, 1);
if (md->virt_addr == 0)
/* no virtual mapping has been installed by the stub */
break;
if (md->virt_addr <= addr &&
(addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT))
return md->phys_addr + addr - md->virt_addr;
}
return addr;
}
static int __init uefi_init(void)
{
efi_char16_t *c16;
void *config_tables;
u64 table_size;
char vendor[100] = "unknown";
int i, retval;
@ -99,7 +121,7 @@ static int __init uefi_init(void)
efi.systab->hdr.revision & 0xffff);
/* Show what we know for posterity */
c16 = early_memremap(efi.systab->fw_vendor,
c16 = early_memremap(efi_to_phys(efi.systab->fw_vendor),
sizeof(vendor));
if (c16) {
for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i)
@ -112,8 +134,14 @@ static int __init uefi_init(void)
efi.systab->hdr.revision >> 16,
efi.systab->hdr.revision & 0xffff, vendor);
retval = efi_config_init(NULL);
table_size = sizeof(efi_config_table_64_t) * efi.systab->nr_tables;
config_tables = early_memremap(efi_to_phys(efi.systab->tables),
table_size);
retval = efi_config_parse_tables(config_tables, efi.systab->nr_tables,
sizeof(efi_config_table_64_t), NULL);
early_memunmap(config_tables, table_size);
out:
early_memunmap(efi.systab, sizeof(efi_system_table_t));
return retval;
@ -163,9 +191,7 @@ static __init void reserve_regions(void)
if (is_normal_ram(md))
early_init_dt_add_memory_arch(paddr, size);
if (is_reserve_region(md) ||
md->type == EFI_BOOT_SERVICES_CODE ||
md->type == EFI_BOOT_SERVICES_DATA) {
if (is_reserve_region(md)) {
memblock_reserve(paddr, size);
if (uefi_debug)
pr_cont("*");
@ -178,123 +204,6 @@ static __init void reserve_regions(void)
set_bit(EFI_MEMMAP, &efi.flags);
}
static u64 __init free_one_region(u64 start, u64 end)
{
u64 size = end - start;
if (uefi_debug)
pr_info(" EFI freeing: 0x%012llx-0x%012llx\n", start, end - 1);
free_bootmem_late(start, size);
return size;
}
static u64 __init free_region(u64 start, u64 end)
{
u64 map_start, map_end, total = 0;
if (end <= start)
return total;
map_start = (u64)memmap.phys_map;
map_end = PAGE_ALIGN(map_start + (memmap.map_end - memmap.map));
map_start &= PAGE_MASK;
if (start < map_end && end > map_start) {
/* region overlaps UEFI memmap */
if (start < map_start)
total += free_one_region(start, map_start);
if (map_end < end)
total += free_one_region(map_end, end);
} else
total += free_one_region(start, end);
return total;
}
static void __init free_boot_services(void)
{
u64 total_freed = 0;
u64 keep_end, free_start, free_end;
efi_memory_desc_t *md;
/*
* If kernel uses larger pages than UEFI, we have to be careful
* not to inadvertantly free memory we want to keep if there is
* overlap at the kernel page size alignment. We do not want to
* free is_reserve_region() memory nor the UEFI memmap itself.
*
* The memory map is sorted, so we keep track of the end of
* any previous region we want to keep, remember any region
* we want to free and defer freeing it until we encounter
* the next region we want to keep. This way, before freeing
* it, we can clip it as needed to avoid freeing memory we
* want to keep for UEFI.
*/
keep_end = 0;
free_start = 0;
for_each_efi_memory_desc(&memmap, md) {
u64 paddr, npages, size;
if (is_reserve_region(md)) {
/*
* We don't want to free any memory from this region.
*/
if (free_start) {
/* adjust free_end then free region */
if (free_end > md->phys_addr)
free_end -= PAGE_SIZE;
total_freed += free_region(free_start, free_end);
free_start = 0;
}
keep_end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT);
continue;
}
if (md->type != EFI_BOOT_SERVICES_CODE &&
md->type != EFI_BOOT_SERVICES_DATA) {
/* no need to free this region */
continue;
}
/*
* We want to free memory from this region.
*/
paddr = md->phys_addr;
npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
if (free_start) {
if (paddr <= free_end)
free_end = paddr + size;
else {
total_freed += free_region(free_start, free_end);
free_start = paddr;
free_end = paddr + size;
}
} else {
free_start = paddr;
free_end = paddr + size;
}
if (free_start < keep_end) {
free_start += PAGE_SIZE;
if (free_start >= free_end)
free_start = 0;
}
}
if (free_start)
total_freed += free_region(free_start, free_end);
if (total_freed)
pr_info("Freed 0x%llx bytes of EFI boot services memory",
total_freed);
}
void __init efi_init(void)
{
struct efi_fdt_params params;
@ -317,159 +226,100 @@ void __init efi_init(void)
return;
reserve_regions();
early_memunmap(memmap.map, params.mmap_size);
}
void __init efi_idmap_init(void)
static bool __init efi_virtmap_init(void)
{
if (!efi_enabled(EFI_BOOT))
return;
efi_memory_desc_t *md;
/* boot time idmap_pg_dir is incomplete, so fill in missing parts */
efi_setup_idmap();
early_memunmap(memmap.map, memmap.map_end - memmap.map);
}
for_each_efi_memory_desc(&memmap, md) {
u64 paddr, npages, size;
pgprot_t prot;
static int __init remap_region(efi_memory_desc_t *md, void **new)
{
u64 paddr, vaddr, npages, size;
if (!(md->attribute & EFI_MEMORY_RUNTIME))
continue;
if (md->virt_addr == 0)
return false;
paddr = md->phys_addr;
npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
paddr = md->phys_addr;
npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
if (is_normal_ram(md))
vaddr = (__force u64)ioremap_cache(paddr, size);
else
vaddr = (__force u64)ioremap(paddr, size);
if (!vaddr) {
pr_err("Unable to remap 0x%llx pages @ %p\n",
npages, (void *)paddr);
return 0;
}
/* adjust for any rounding when EFI and system pagesize differs */
md->virt_addr = vaddr + (md->phys_addr - paddr);
if (uefi_debug)
pr_info(" EFI remap 0x%012llx => %p\n",
pr_info(" EFI remap 0x%016llx => %p\n",
md->phys_addr, (void *)md->virt_addr);
memcpy(*new, md, memmap.desc_size);
*new += memmap.desc_size;
/*
* Only regions of type EFI_RUNTIME_SERVICES_CODE need to be
* executable, everything else can be mapped with the XN bits
* set.
*/
if (!is_normal_ram(md))
prot = __pgprot(PROT_DEVICE_nGnRE);
else if (md->type == EFI_RUNTIME_SERVICES_CODE)
prot = PAGE_KERNEL_EXEC;
else
prot = PAGE_KERNEL;
return 1;
create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot);
}
return true;
}
/*
* Switch UEFI from an identity map to a kernel virtual map
* Enable the UEFI Runtime Services if all prerequisites are in place, i.e.,
* non-early mapping of the UEFI system table and virtual mappings for all
* EFI_MEMORY_RUNTIME regions.
*/
static int __init arm64_enter_virtual_mode(void)
static int __init arm64_enable_runtime_services(void)
{
efi_memory_desc_t *md;
phys_addr_t virtmap_phys;
void *virtmap, *virt_md;
efi_status_t status;
u64 mapsize;
int count = 0;
unsigned long flags;
if (!efi_enabled(EFI_BOOT)) {
pr_info("EFI services will not be available.\n");
return -1;
}
mapsize = memmap.map_end - memmap.map;
if (efi_runtime_disabled()) {
pr_info("EFI runtime services will be disabled.\n");
return -1;
}
pr_info("Remapping and enabling EFI services.\n");
/* replace early memmap mapping with permanent mapping */
mapsize = memmap.map_end - memmap.map;
memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map,
mapsize);
memmap.map_end = memmap.map + mapsize;
efi.memmap = &memmap;
/* Map the runtime regions */
virtmap = kmalloc(mapsize, GFP_KERNEL);
if (!virtmap) {
pr_err("Failed to allocate EFI virtual memmap\n");
if (!memmap.map) {
pr_err("Failed to remap EFI memory map\n");
return -1;
}
virtmap_phys = virt_to_phys(virtmap);
virt_md = virtmap;
memmap.map_end = memmap.map + mapsize;
efi.memmap = &memmap;
for_each_efi_memory_desc(&memmap, md) {
if (!(md->attribute & EFI_MEMORY_RUNTIME))
continue;
if (!remap_region(md, &virt_md))
goto err_unmap;
++count;
}
efi.systab = (__force void *)efi_lookup_mapped_addr(efi_system_table);
efi.systab = (__force void *)ioremap_cache(efi_system_table,
sizeof(efi_system_table_t));
if (!efi.systab) {
/*
* If we have no virtual mapping for the System Table at this
* point, the memory map doesn't cover the physical offset where
* it resides. This means the System Table will be inaccessible
* to Runtime Services themselves once the virtual mapping is
* installed.
*/
pr_err("Failed to remap EFI System Table -- buggy firmware?\n");
goto err_unmap;
pr_err("Failed to remap EFI System Table\n");
return -1;
}
set_bit(EFI_SYSTEM_TABLES, &efi.flags);
local_irq_save(flags);
cpu_switch_mm(idmap_pg_dir, &init_mm);
/* Call SetVirtualAddressMap with the physical address of the map */
runtime = efi.systab->runtime;
efi.set_virtual_address_map = runtime->set_virtual_address_map;
status = efi.set_virtual_address_map(count * memmap.desc_size,
memmap.desc_size,
memmap.desc_version,
(efi_memory_desc_t *)virtmap_phys);
cpu_set_reserved_ttbr0();
flush_tlb_all();
local_irq_restore(flags);
kfree(virtmap);
free_boot_services();
if (status != EFI_SUCCESS) {
pr_err("Failed to set EFI virtual address map! [%lx]\n",
status);
if (!efi_virtmap_init()) {
pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n");
return -1;
}
/* Set up runtime services function pointers */
runtime = efi.systab->runtime;
efi_native_runtime_setup();
set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
efi.runtime_version = efi.systab->hdr.revision;
return 0;
err_unmap:
/* unmap all mappings that succeeded: there are 'count' of those */
for (virt_md = virtmap; count--; virt_md += memmap.desc_size) {
md = virt_md;
iounmap((__force void __iomem *)md->virt_addr);
}
kfree(virtmap);
return -1;
}
early_initcall(arm64_enter_virtual_mode);
early_initcall(arm64_enable_runtime_services);
static int __init arm64_dmi_init(void)
{
@ -484,3 +334,23 @@ static int __init arm64_dmi_init(void)
return 0;
}
core_initcall(arm64_dmi_init);
static void efi_set_pgd(struct mm_struct *mm)
{
cpu_switch_mm(mm->pgd, mm);
flush_tlb_all();
if (icache_is_aivivt())
__flush_icache_all();
}
void efi_virtmap_load(void)
{
preempt_disable();
efi_set_pgd(&efi_mm);
}
void efi_virtmap_unload(void)
{
efi_set_pgd(current->active_mm);
preempt_enable();
}

66
arch/arm64/kernel/entry.S
View File

@ -269,18 +269,18 @@ ENDPROC(el1_error_invalid)
el1_sync:
kernel_entry 1
mrs x1, esr_el1 // read the syndrome register
lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class
cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1
lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
b.eq el1_da
cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
b.eq el1_undef
cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
b.eq el1_sp_pc
cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
b.eq el1_sp_pc
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
b.eq el1_undef
cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1
cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
b.ge el1_dbg
b el1_inv
el1_da:
@ -318,7 +318,7 @@ el1_dbg:
/*
* Debug exception handling
*/
cmp x24, #ESR_EL1_EC_BRK64 // if BRK64
cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
cinc x24, x24, eq // set bit '0'
tbz x24, #0, el1_inv // EL1 only
mrs x0, far_el1
@ -375,26 +375,26 @@ el1_preempt:
el0_sync:
kernel_entry 0
mrs x25, esr_el1 // read the syndrome register
lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
b.eq el0_svc
cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
b.eq el0_da
cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
b.eq el0_ia
cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
b.eq el0_fpsimd_acc
cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception
cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
b.eq el0_fpsimd_exc
cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
b.eq el0_undef
cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
b.eq el0_sp_pc
cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
b.eq el0_sp_pc
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
b.eq el0_undef
cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
b.ge el0_dbg
b el0_inv
@ -403,37 +403,37 @@ el0_sync:
el0_sync_compat:
kernel_entry 0, 32
mrs x25, esr_el1 // read the syndrome register
lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
b.eq el0_svc_compat
cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
b.eq el0_da
cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
b.eq el0_ia
cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
b.eq el0_fpsimd_acc
cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception
cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
b.eq el0_fpsimd_exc
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap
cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap
cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap
cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap
cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap
cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
b.eq el0_undef
cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
b.ge el0_dbg
b el0_inv
el0_svc_compat:
/*
* AArch32 syscall handling
*/
adr stbl, compat_sys_call_table // load compat syscall table pointer
adrp stbl, compat_sys_call_table // load compat syscall table pointer
uxtw scno, w7 // syscall number in w7 (r7)
mov sc_nr, #__NR_compat_syscalls
b el0_svc_naked

34
arch/arm64/kernel/sys32.S → arch/arm64/kernel/entry32.S
View File

@ -27,26 +27,26 @@
* System call wrappers for the AArch32 compatibility layer.
*/
compat_sys_sigreturn_wrapper:
ENTRY(compat_sys_sigreturn_wrapper)
mov x0, sp
mov x27, #0 // prevent syscall restart handling (why)
b compat_sys_sigreturn
ENDPROC(compat_sys_sigreturn_wrapper)
compat_sys_rt_sigreturn_wrapper:
ENTRY(compat_sys_rt_sigreturn_wrapper)
mov x0, sp
mov x27, #0 // prevent syscall restart handling (why)
b compat_sys_rt_sigreturn
ENDPROC(compat_sys_rt_sigreturn_wrapper)
compat_sys_statfs64_wrapper:
ENTRY(compat_sys_statfs64_wrapper)
mov w3, #84
cmp w1, #88
csel w1, w3, w1, eq
b compat_sys_statfs64
ENDPROC(compat_sys_statfs64_wrapper)
compat_sys_fstatfs64_wrapper:
ENTRY(compat_sys_fstatfs64_wrapper)
mov w3, #84
cmp w1, #88
csel w1, w3, w1, eq
@ -58,33 +58,33 @@ ENDPROC(compat_sys_fstatfs64_wrapper)
* in registers or that take 32-bit parameters which require sign
* extension.
*/
compat_sys_pread64_wrapper:
ENTRY(compat_sys_pread64_wrapper)
regs_to_64 x3, x4, x5
b sys_pread64
ENDPROC(compat_sys_pread64_wrapper)
compat_sys_pwrite64_wrapper:
ENTRY(compat_sys_pwrite64_wrapper)
regs_to_64 x3, x4, x5
b sys_pwrite64
ENDPROC(compat_sys_pwrite64_wrapper)
compat_sys_truncate64_wrapper:
ENTRY(compat_sys_truncate64_wrapper)
regs_to_64 x1, x2, x3
b sys_truncate
ENDPROC(compat_sys_truncate64_wrapper)
compat_sys_ftruncate64_wrapper:
ENTRY(compat_sys_ftruncate64_wrapper)
regs_to_64 x1, x2, x3
b sys_ftruncate
ENDPROC(compat_sys_ftruncate64_wrapper)
compat_sys_readahead_wrapper:
ENTRY(compat_sys_readahead_wrapper)
regs_to_64 x1, x2, x3
mov w2, w4
b sys_readahead
ENDPROC(compat_sys_readahead_wrapper)
compat_sys_fadvise64_64_wrapper:
ENTRY(compat_sys_fadvise64_64_wrapper)
mov w6, w1
regs_to_64 x1, x2, x3
regs_to_64 x2, x4, x5
@ -92,24 +92,14 @@ compat_sys_fadvise64_64_wrapper:
b sys_fadvise64_64
ENDPROC(compat_sys_fadvise64_64_wrapper)
compat_sys_sync_file_range2_wrapper:
ENTRY(compat_sys_sync_file_range2_wrapper)
regs_to_64 x2, x2, x3
regs_to_64 x3, x4, x5
b sys_sync_file_range2
ENDPROC(compat_sys_sync_file_range2_wrapper)
compat_sys_fallocate_wrapper:
ENTRY(compat_sys_fallocate_wrapper)
regs_to_64 x2, x2, x3
regs_to_64 x3, x4, x5
b sys_fallocate
ENDPROC(compat_sys_fallocate_wrapper)
#undef __SYSCALL
#define __SYSCALL(x, y) .quad y // x
/*
* The system calls table must be 4KB aligned.
*/
.align 12
ENTRY(compat_sys_call_table)
#include <asm/unistd32.h>

2
arch/arm64/kernel/hw_breakpoint.c
View File

@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = {
.notifier_call = hw_breakpoint_reset_notify,
};
#ifdef CONFIG_ARM64_CPU_SUSPEND
#ifdef CONFIG_CPU_PM
extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
#else
static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))

47
arch/arm64/kernel/insn.c
View File

@ -17,14 +17,19 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/stop_machine.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/debug-monitors.h>
#include <asm/fixmap.h>
#include <asm/insn.h>
#define AARCH64_INSN_SF_BIT BIT(31)
@ -72,6 +77,29 @@ bool __kprobes aarch64_insn_is_nop(u32 insn)
}
}
static DEFINE_SPINLOCK(patch_lock);
static void __kprobes *patch_map(void *addr, int fixmap)
{
unsigned long uintaddr = (uintptr_t) addr;
bool module = !core_kernel_text(uintaddr);
struct page *page;
if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
page = vmalloc_to_page(addr);
else
page = virt_to_page(addr);
BUG_ON(!page);
set_fixmap(fixmap, page_to_phys(page));
return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
}
static void __kprobes patch_unmap(int fixmap)
{
clear_fixmap(fixmap);
}
/*
* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
* little-endian.
@ -88,10 +116,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
return ret;
}
static int __kprobes __aarch64_insn_write(void *addr, u32 insn)
{
void *waddr = addr;
unsigned long flags = 0;
int ret;
spin_lock_irqsave(&patch_lock, flags);
waddr = patch_map(addr, FIX_TEXT_POKE0);
ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE);
patch_unmap(FIX_TEXT_POKE0);
spin_unlock_irqrestore(&patch_lock, flags);
return ret;
}
int __kprobes aarch64_insn_write(void *addr, u32 insn)
{
insn = cpu_to_le32(insn);
return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
return __aarch64_insn_write(addr, insn);
}
static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)

2
arch/arm64/kernel/psci.c
View File

@ -540,8 +540,6 @@ const struct cpu_operations cpu_psci_ops = {
.name = "psci",
#ifdef CONFIG_CPU_IDLE
.cpu_init_idle = cpu_psci_cpu_init_idle,
#endif
#ifdef CONFIG_ARM64_CPU_SUSPEND
.cpu_suspend = cpu_psci_cpu_suspend,
#endif
#ifdef CONFIG_SMP

22
arch/arm64/kernel/setup.c
View File

@ -40,6 +40,7 @@
#include <linux/fs.h>
#include <linux/proc_fs.h>
#include <linux/memblock.h>
#include <linux/of_iommu.h>
#include <linux/of_fdt.h>
#include <linux/of_platform.h>
#include <linux/efi.h>
@ -322,25 +323,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys)
dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name());
}
/*
* Limit the memory size that was specified via FDT.
*/
static int __init early_mem(char *p)
{
phys_addr_t limit;
if (!p)
return 1;
limit = memparse(p, &p) & PAGE_MASK;
pr_notice("Memory limited to %lldMB\n", limit >> 20);
memblock_enforce_memory_limit(limit);
return 0;
}
early_param("mem", early_mem);
static void __init request_standard_resources(void)
{
struct memblock_region *region;
@ -401,7 +383,6 @@ void __init setup_arch(char **cmdline_p)
paging_init();
request_standard_resources();
efi_idmap_init();
early_ioremap_reset();
unflatten_device_tree();
@ -425,6 +406,7 @@ void __init setup_arch(char **cmdline_p)
static int __init arm64_device_init(void)
{
of_iommu_init();
of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
return 0;
}

7
arch/arm64/kernel/signal32.c
View File

@ -440,7 +440,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
{
compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
compat_ulong_t retcode;
compat_ulong_t spsr = regs->pstate & ~PSR_f;
compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT);
int thumb;
/* Check if the handler is written for ARM or Thumb */
@ -454,6 +454,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
/* The IT state must be cleared for both ARM and Thumb-2 */
spsr &= ~COMPAT_PSR_IT_MASK;
/* Restore the original endianness */
spsr |= COMPAT_PSR_ENDSTATE;
if (ka->sa.sa_flags & SA_RESTORER) {
retcode = ptr_to_compat(ka->sa.sa_restorer);
} else {
@ -501,7 +504,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf,
__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);
/* set the compat FSR WnR */
__put_user_error(!!(current->thread.fault_code & ESR_EL1_WRITE) <<
__put_user_error(!!(current->thread.fault_code & ESR_ELx_WNR) <<
FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err);
__put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err);
__put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err);

10
arch/arm64/kernel/smp.c
View File

@ -65,7 +65,6 @@ struct secondary_data secondary_data;
enum ipi_msg_type {
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CALL_FUNC_SINGLE,
IPI_CPU_STOP,
IPI_TIMER,
IPI_IRQ_WORK,
@ -483,7 +482,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s) [x] = s
S(IPI_RESCHEDULE, "Rescheduling interrupts"),
S(IPI_CALL_FUNC, "Function call interrupts"),
S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
S(IPI_CPU_STOP, "CPU stop interrupts"),
S(IPI_TIMER, "Timer broadcast interrupts"),
S(IPI_IRQ_WORK, "IRQ work interrupts"),
@ -527,7 +525,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
void arch_send_call_function_single_ipi(int cpu)
{
smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
}
#ifdef CONFIG_IRQ_WORK
@ -585,12 +583,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
irq_exit();
break;
case IPI_CALL_FUNC_SINGLE:
irq_enter();
generic_smp_call_function_single_interrupt();
irq_exit();
break;
case IPI_CPU_STOP:
irq_enter();
ipi_cpu_stop(cpu);

21
arch/arm64/kernel/suspend.c
View File

@ -1,7 +1,6 @@
#include <linux/percpu.h>
#include <linux/slab.h>
#include <asm/cacheflush.h>
#include <asm/cpu_ops.h>
#include <asm/debug-monitors.h>
#include <asm/pgtable.h>
#include <asm/memory.h>
@ -51,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
hw_breakpoint_restore = hw_bp_restore;
}
/**
* cpu_suspend() - function to enter a low-power state
* @arg: argument to pass to CPU suspend operations
*
* Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
* operations back-end error code otherwise.
*/
int cpu_suspend(unsigned long arg)
{
int cpu = smp_processor_id();
/*
* If cpu_ops have not been registered or suspend
* has not been initialized, cpu_suspend call fails early.
*/
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
return -EOPNOTSUPP;
return cpu_ops[cpu]->cpu_suspend(arg);
}
/*
* __cpu_suspend
*

5
arch/arm64/kernel/sys.c
View File

@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
/*
* Wrappers to pass the pt_regs argument.
*/
asmlinkage long sys_rt_sigreturn_wrapper(void);
#define sys_rt_sigreturn sys_rt_sigreturn_wrapper
#include <asm/syscalls.h>
#undef __SYSCALL
#define __SYSCALL(nr, sym) [nr] = sym,
@ -50,7 +49,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
* The sys_call_table array must be 4K aligned to be accessible from
* kernel/entry.S.
*/
void *sys_call_table[__NR_syscalls] __aligned(4096) = {
void * const sys_call_table[__NR_syscalls] __aligned(4096) = {
[0 ... __NR_syscalls - 1] = sys_ni_syscall,
#include <asm/unistd.h>
};

51
arch/arm64/kernel/sys32.c Normal file
View File

@ -0,0 +1,51 @@
/*
* arch/arm64/kernel/sys32.c
*
* Copyright (C) 2015 ARM Ltd.
*
* This program is free software(void); you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http(void);//www.gnu.org/licenses/>.
*/
/*
* Needed to avoid conflicting __NR_* macros between uapi/asm/unistd.h and
* asm/unistd32.h.
*/
#define __COMPAT_SYSCALL_NR
#include <linux/compiler.h>
#include <linux/syscalls.h>
asmlinkage long compat_sys_sigreturn_wrapper(void);
asmlinkage long compat_sys_rt_sigreturn_wrapper(void);
asmlinkage long compat_sys_statfs64_wrapper(void);
asmlinkage long compat_sys_fstatfs64_wrapper(void);
asmlinkage long compat_sys_pread64_wrapper(void);
asmlinkage long compat_sys_pwrite64_wrapper(void);
asmlinkage long compat_sys_truncate64_wrapper(void);
asmlinkage long compat_sys_ftruncate64_wrapper(void);
asmlinkage long compat_sys_readahead_wrapper(void);
asmlinkage long compat_sys_fadvise64_64_wrapper(void);
asmlinkage long compat_sys_sync_file_range2_wrapper(void);
asmlinkage long compat_sys_fallocate_wrapper(void);
#undef __SYSCALL
#define __SYSCALL(nr, sym) [nr] = sym,
/*
* The sys_call_table array must be 4K aligned to be accessible from
* kernel/entry.S.
*/
void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = {
[0 ... __NR_compat_syscalls - 1] = sys_ni_syscall,
#include <asm/unistd32.h>
};

50
arch/arm64/kernel/traps.c
View File

@ -33,6 +33,7 @@
#include <asm/atomic.h>
#include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/traps.h>
#include <asm/stacktrace.h>
#include <asm/exception.h>
@ -373,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs)
return sys_ni_syscall();
}
static const char *esr_class_str[] = {
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
[ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
[ESR_ELx_EC_WFx] = "WFI/WFE",
[ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
[ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
[ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
[ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
[ESR_ELx_EC_FP_ASIMD] = "ASIMD",
[ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
[ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
[ESR_ELx_EC_ILL] = "PSTATE.IL",
[ESR_ELx_EC_SVC32] = "SVC (AArch32)",
[ESR_ELx_EC_HVC32] = "HVC (AArch32)",
[ESR_ELx_EC_SMC32] = "SMC (AArch32)",
[ESR_ELx_EC_SVC64] = "SVC (AArch64)",
[ESR_ELx_EC_HVC64] = "HVC (AArch64)",
[ESR_ELx_EC_SMC64] = "SMC (AArch64)",
[ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
[ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
[ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
[ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
[ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
[ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
[ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
[ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
[ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
[ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
[ESR_ELx_EC_SERROR] = "SError",
[ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
[ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
[ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
[ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
[ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
[ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
[ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
[ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
[ESR_ELx_EC_BRK64] = "BRK (AArch64)",
};
const char *esr_get_class_string(u32 esr)
{
return esr_class_str[esr >> ESR_ELx_EC_SHIFT];
}
/*
* bad_mode handles the impossible case in the exception vector.
*/
@ -382,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
void __user *pc = (void __user *)instruction_pointer(regs);
console_verbose();
pr_crit("Bad mode in %s handler detected, code 0x%08x\n",
handler[reason], esr);
pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n",
handler[reason], esr, esr_get_class_string(esr));
__show_regs(regs);
info.si_signo = SIGILL;

17
arch/arm64/kernel/vmlinux.lds.S
View File

@ -8,6 +8,7 @@
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include "image.h"
@ -49,6 +50,14 @@ PECOFF_FILE_ALIGNMENT = 0x200;
#define PECOFF_EDATA_PADDING
#endif
#ifdef CONFIG_DEBUG_ALIGN_RODATA
#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT);
#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO
#else
#define ALIGN_DEBUG_RO
#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min);
#endif
SECTIONS
{
/*
@ -71,6 +80,7 @@ SECTIONS
_text = .;
HEAD_TEXT
}
ALIGN_DEBUG_RO
.text : { /* Real text segment */
_stext = .; /* Text and read-only data */
__exception_text_start = .;
@ -87,19 +97,22 @@ SECTIONS
*(.got) /* Global offset table */
}
ALIGN_DEBUG_RO
RO_DATA(PAGE_SIZE)
EXCEPTION_TABLE(8)
NOTES
ALIGN_DEBUG_RO
_etext = .; /* End of text and rodata section */
. = ALIGN(PAGE_SIZE);
ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
__init_begin = .;
INIT_TEXT_SECTION(8)
.exit.text : {
ARM_EXIT_KEEP(EXIT_TEXT)
}
. = ALIGN(16);
ALIGN_DEBUG_RO_MIN(16)
.init.data : {
INIT_DATA
INIT_SETUP(16)

5
arch/arm64/kvm/emulate.c
View File

@ -22,6 +22,7 @@
*/
#include <linux/kvm_host.h>
#include <asm/esr.h>
#include <asm/kvm_emulate.h>
/*
@ -55,8 +56,8 @@ static int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
{
u32 esr = kvm_vcpu_get_hsr(vcpu);
if (esr & ESR_EL2_CV)
return (esr & ESR_EL2_COND) >> ESR_EL2_COND_SHIFT;
if (esr & ESR_ELx_CV)
return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
return -1;
}

39
arch/arm64/kvm/handle_exit.c
View File

@ -21,8 +21,10 @@
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/esr.h>
#include <asm/kvm_coproc.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_psci.h>
@ -61,7 +63,7 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/
static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
if (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EC_WFI_ISS_WFE)
if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE)
kvm_vcpu_on_spin(vcpu);
else
kvm_vcpu_block(vcpu);
@ -72,29 +74,30 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
}
static exit_handle_fn arm_exit_handlers[] = {
[ESR_EL2_EC_WFI] = kvm_handle_wfx,
[ESR_EL2_EC_CP15_32] = kvm_handle_cp15_32,
[ESR_EL2_EC_CP15_64] = kvm_handle_cp15_64,
[ESR_EL2_EC_CP14_MR] = kvm_handle_cp14_32,
[ESR_EL2_EC_CP14_LS] = kvm_handle_cp14_load_store,
[ESR_EL2_EC_CP14_64] = kvm_handle_cp14_64,
[ESR_EL2_EC_HVC32] = handle_hvc,
[ESR_EL2_EC_SMC32] = handle_smc,
[ESR_EL2_EC_HVC64] = handle_hvc,
[ESR_EL2_EC_SMC64] = handle_smc,
[ESR_EL2_EC_SYS64] = kvm_handle_sys_reg,
[ESR_EL2_EC_IABT] = kvm_handle_guest_abort,
[ESR_EL2_EC_DABT] = kvm_handle_guest_abort,
[ESR_ELx_EC_WFx] = kvm_handle_wfx,
[ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32,
[ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64,
[ESR_ELx_EC_CP14_MR] = kvm_handle_cp14_32,
[ESR_ELx_EC_CP14_LS] = kvm_handle_cp14_load_store,
[ESR_ELx_EC_CP14_64] = kvm_handle_cp14_64,
[ESR_ELx_EC_HVC32] = handle_hvc,
[ESR_ELx_EC_SMC32] = handle_smc,
[ESR_ELx_EC_HVC64] = handle_hvc,
[ESR_ELx_EC_SMC64] = handle_smc,
[ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
[ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
[ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
};
static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
{
u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
u32 hsr = kvm_vcpu_get_hsr(vcpu);
u8 hsr_ec = hsr >> ESR_ELx_EC_SHIFT;
if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) ||
!arm_exit_handlers[hsr_ec]) {
kvm_err("Unknown exception class: hsr: %#08x\n",
(unsigned int)kvm_vcpu_get_hsr(vcpu));
kvm_err("Unknown exception class: hsr: %#08x -- %s\n",
hsr, esr_get_class_string(hsr));
BUG();
}

17
arch/arm64/kvm/hyp.S
View File

@ -17,15 +17,16 @@
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/memory.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/fpsimdmacros.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmu.h>
#include <asm/memory.h>
#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
@ -1141,9 +1142,9 @@ el1_sync: // Guest trapped into EL2
push x2, x3
mrs x1, esr_el2
lsr x2, x1, #ESR_EL2_EC_SHIFT
lsr x2, x1, #ESR_ELx_EC_SHIFT
cmp x2, #ESR_EL2_EC_HVC64
cmp x2, #ESR_ELx_EC_HVC64
b.ne el1_trap
mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
@ -1178,13 +1179,13 @@ el1_trap:
* x1: ESR
* x2: ESR_EC
*/
cmp x2, #ESR_EL2_EC_DABT
mov x0, #ESR_EL2_EC_IABT
cmp x2, #ESR_ELx_EC_DABT_LOW
mov x0, #ESR_ELx_EC_IABT_LOW
ccmp x2, x0, #4, ne
b.ne 1f // Not an abort we care about
/* This is an abort. Check for permission fault */
and x2, x1, #ESR_EL2_FSC_TYPE
and x2, x1, #ESR_ELx_FSC_TYPE
cmp x2, #FSC_PERM
b.ne 1f // Not a permission fault

14
arch/arm64/kvm/inject_fault.c
View File

@ -118,27 +118,27 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
* instruction set. Report an external synchronous abort.
*/
if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_EL1_IL;
esr |= ESR_ELx_IL;
/*
* Here, the guest runs in AArch64 mode when in EL1. If we get
* an AArch32 fault, it means we managed to trap an EL0 fault.
*/
if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
esr |= (ESR_EL1_EC_IABT_EL0 << ESR_EL1_EC_SHIFT);
esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
else
esr |= (ESR_EL1_EC_IABT_EL1 << ESR_EL1_EC_SHIFT);
esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
if (!is_iabt)
esr |= ESR_EL1_EC_DABT_EL0;
esr |= ESR_ELx_EC_DABT_LOW;
vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_EL2_EC_xABT_xFSR_EXTABT;
vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_ELx_FSC_EXTABT;
}
static void inject_undef64(struct kvm_vcpu *vcpu)
{
unsigned long cpsr = *vcpu_cpsr(vcpu);
u32 esr = (ESR_EL1_EC_UNKNOWN << ESR_EL1_EC_SHIFT);
u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
*vcpu_spsr(vcpu) = cpsr;
*vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu);
@ -151,7 +151,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
* set.
*/
if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_EL1_IL;
esr |= ESR_ELx_IL;
vcpu_sys_reg(vcpu, ESR_EL1) = esr;
}

23
arch/arm64/kvm/sys_regs.c
View File

@ -20,17 +20,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/mm.h>
#include <linux/kvm_host.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_coproc.h>
#include <asm/kvm_mmu.h>
#include <asm/cacheflush.h>
#include <asm/cputype.h>
#include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_coproc.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_host.h>
#include <asm/kvm_mmu.h>
#include <trace/events/kvm.h>
#include "sys_regs.h"
@ -760,12 +763,12 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu,
int cp;
switch(hsr_ec) {
case ESR_EL2_EC_CP15_32:
case ESR_EL2_EC_CP15_64:
case ESR_ELx_EC_CP15_32:
case ESR_ELx_EC_CP15_64:
cp = 15;
break;
case ESR_EL2_EC_CP14_MR:
case ESR_EL2_EC_CP14_64:
case ESR_ELx_EC_CP14_MR:
case ESR_ELx_EC_CP14_64:
cp = 14;
break;
default:

116
arch/arm64/mm/dma-mapping.c
View File

@ -134,16 +134,17 @@ static void __dma_free_coherent(struct device *dev, size_t size,
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
}
static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs)
static void *__dma_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs)
{
struct page *page;
void *ptr, *coherent_ptr;
bool coherent = is_device_dma_coherent(dev);
size = PAGE_ALIGN(size);
if (!(flags & __GFP_WAIT)) {
if (!coherent && !(flags & __GFP_WAIT)) {
struct page *page = NULL;
void *addr = __alloc_from_pool(size, &page);
@ -151,13 +152,16 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
*dma_handle = phys_to_dma(dev, page_to_phys(page));
return addr;
}
ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
if (!ptr)
goto no_mem;
/* no need for non-cacheable mapping if coherent */
if (coherent)
return ptr;
/* remove any dirty cache lines on the kernel alias */
__dma_flush_range(ptr, ptr + size);
@ -179,15 +183,17 @@ no_mem:
return NULL;
}
static void __dma_free_noncoherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
struct dma_attrs *attrs)
static void __dma_free(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
struct dma_attrs *attrs)
{
void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
if (__free_from_pool(vaddr, size))
return;
vunmap(vaddr);
if (!is_device_dma_coherent(dev)) {
if (__free_from_pool(vaddr, size))
return;
vunmap(vaddr);
}
__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
}
@ -199,7 +205,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
dma_addr_t dev_addr;
dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
if (!is_device_dma_coherent(dev))
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
return dev_addr;
}
@ -209,7 +216,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
if (!is_device_dma_coherent(dev))
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
}
@ -221,9 +229,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
int i, ret;
ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
for_each_sg(sgl, sg, ret, i)
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
if (!is_device_dma_coherent(dev))
for_each_sg(sgl, sg, ret, i)
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
return ret;
}
@ -236,9 +245,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
if (!is_device_dma_coherent(dev))
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
}
@ -246,7 +256,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev,
dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir)
{
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
if (!is_device_dma_coherent(dev))
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
}
@ -255,7 +266,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev,
enum dma_data_direction dir)
{
swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
if (!is_device_dma_coherent(dev))
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
}
static void __swiotlb_sync_sg_for_cpu(struct device *dev,
@ -265,9 +277,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev,
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
if (!is_device_dma_coherent(dev))
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
}
@ -279,9 +292,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
int i;
swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
for_each_sg(sgl, sg, nelems, i)
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
if (!is_device_dma_coherent(dev))
for_each_sg(sgl, sg, nelems, i)
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
}
/* vma->vm_page_prot must be set appropriately before calling this function */
@ -308,28 +322,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
return ret;
}
static int __swiotlb_mmap_noncoherent(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
struct dma_attrs *attrs)
static int __swiotlb_mmap(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
struct dma_attrs *attrs)
{
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
is_device_dma_coherent(dev));
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}
static int __swiotlb_mmap_coherent(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
struct dma_attrs *attrs)
{
/* Just use whatever page_prot attributes were specified */
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}
struct dma_map_ops noncoherent_swiotlb_dma_ops = {
.alloc = __dma_alloc_noncoherent,
.free = __dma_free_noncoherent,
.mmap = __swiotlb_mmap_noncoherent,
static struct dma_map_ops swiotlb_dma_ops = {
.alloc = __dma_alloc,
.free = __dma_free,
.mmap = __swiotlb_mmap,
.map_page = __swiotlb_map_page,
.unmap_page = __swiotlb_unmap_page,
.map_sg = __swiotlb_map_sg_attrs,
@ -341,24 +347,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = {
.dma_supported = swiotlb_dma_supported,
.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
struct dma_map_ops coherent_swiotlb_dma_ops = {
.alloc = __dma_alloc_coherent,
.free = __dma_free_coherent,
.mmap = __swiotlb_mmap_coherent,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.dma_supported = swiotlb_dma_supported,
.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
extern int swiotlb_late_init_with_default_size(size_t default_size);
@ -427,7 +415,7 @@ static int __init swiotlb_late_init(void)
{
size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
dma_ops = &noncoherent_swiotlb_dma_ops;
dma_ops = &swiotlb_dma_ops;
return swiotlb_late_init_with_default_size(swiotlb_size);
}

30
arch/arm64/mm/dump.c
View File

@ -14,14 +14,18 @@
* of the License.
*/
#include <linux/debugfs.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/io.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <asm/fixmap.h>
#include <asm/memory.h>
#include <asm/pgtable.h>
#include <asm/pgtable-hwdef.h>
#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS)
@ -37,10 +41,10 @@ enum address_markers_idx {
VMEMMAP_START_NR,
VMEMMAP_END_NR,
#endif
PCI_START_NR,
PCI_END_NR,
FIXADDR_START_NR,
FIXADDR_END_NR,
PCI_START_NR,
PCI_END_NR,
MODULES_START_NR,
MODUELS_END_NR,
KERNEL_SPACE_NR,
@ -53,10 +57,10 @@ static struct addr_marker address_markers[] = {
{ 0, "vmemmap start" },
{ 0, "vmemmap end" },
#endif
{ (unsigned long) PCI_IOBASE, "PCI I/O start" },
{ (unsigned long) PCI_IOBASE + SZ_16M, "PCI I/O end" },
{ FIXADDR_START, "Fixmap start" },
{ FIXADDR_TOP, "Fixmap end" },
{ PCI_IO_START, "PCI I/O start" },
{ PCI_IO_END, "PCI I/O end" },
{ MODULES_VADDR, "Modules start" },
{ MODULES_END, "Modules end" },
{ PAGE_OFFSET, "Kernel Mapping" },
@ -246,10 +250,12 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
addr = start + i * PMD_SIZE;
if (pmd_none(*pmd) || pmd_sect(*pmd) || pmd_bad(*pmd))
if (pmd_none(*pmd) || pmd_sect(*pmd)) {
note_page(st, addr, 3, pmd_val(*pmd));
else
} else {
BUG_ON(pmd_bad(*pmd));
walk_pte(st, pmd, addr);
}
}
}
@ -261,10 +267,12 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
addr = start + i * PUD_SIZE;
if (pud_none(*pud) || pud_sect(*pud) || pud_bad(*pud))
if (pud_none(*pud) || pud_sect(*pud)) {
note_page(st, addr, 2, pud_val(*pud));
else
} else {
BUG_ON(pud_bad(*pud));
walk_pmd(st, pud, addr);
}
}
}
@ -276,10 +284,12 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st
for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
addr = start + i * PGDIR_SIZE;
if (pgd_none(*pgd) || pgd_bad(*pgd))
if (pgd_none(*pgd)) {
note_page(st, addr, 1, pgd_val(*pgd));
else
} else {
BUG_ON(pgd_bad(*pgd));
walk_pud(st, pgd, addr);
}
}
}

2
arch/arm64/mm/fault.c
View File

@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
if (esr & ESR_LNX_EXEC) {
vm_flags = VM_EXEC;
} else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) {
} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
vm_flags = VM_WRITE;
mm_flags |= FAULT_FLAG_WRITE;
}

25
arch/arm64/mm/init.c
View File

@ -35,6 +35,7 @@
#include <linux/efi.h>
#include <asm/fixmap.h>
#include <asm/memory.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
@ -136,10 +137,29 @@ static void arm64_memory_present(void)
}
#endif
static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
/*
* Limit the memory size that was specified via FDT.
*/
static int __init early_mem(char *p)
{
if (!p)
return 1;
memory_limit = memparse(p, &p) & PAGE_MASK;
pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
return 0;
}
early_param("mem", early_mem);
void __init arm64_memblock_init(void)
{
phys_addr_t dma_phys_limit = 0;
memblock_enforce_memory_limit(memory_limit);
/*
* Register the kernel text, kernel data, initrd, and initial
* pagetables with memblock.
@ -277,8 +297,8 @@ void __init mem_init(void)
" vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
" 0x%16lx - 0x%16lx (%6ld MB actual)\n"
#endif
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
" fixed : 0x%16lx - 0x%16lx (%6ld KB)\n"
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
" modules : 0x%16lx - 0x%16lx (%6ld MB)\n"
" memory : 0x%16lx - 0x%16lx (%6ld MB)\n"
" .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
@ -291,8 +311,8 @@ void __init mem_init(void)
MLM((unsigned long)virt_to_page(PAGE_OFFSET),
(unsigned long)virt_to_page(high_memory)),
#endif
MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M),
MLK(FIXADDR_START, FIXADDR_TOP),
MLM(PCI_IO_START, PCI_IO_END),
MLM(MODULES_VADDR, MODULES_END),
MLM(PAGE_OFFSET, (unsigned long)high_memory),
MLK_ROUNDUP(__init_begin, __init_end),
@ -325,6 +345,7 @@ void __init mem_init(void)
void free_initmem(void)
{
fixup_init();
free_initmem_default(0);
free_alternatives_memory();
}

1
arch/arm64/mm/ioremap.c
View File

@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
if (!area)
return NULL;
addr = (unsigned long)area->addr;
area->phys_addr = phys_addr;
err = ioremap_page_range(addr, addr + size, phys_addr, prot);
if (err) {

2
arch/arm64/mm/mm.h
View File

@ -1 +1,3 @@
extern void __init bootmem_init(void);
void fixup_init(void);

342
arch/arm64/mm/mmu.c
View File

@ -26,6 +26,8 @@
#include <linux/memblock.h>
#include <linux/fs.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/stop_machine.h>
#include <asm/cputype.h>
#include <asm/fixmap.h>
@ -45,80 +47,6 @@
struct page *empty_zero_page;
EXPORT_SYMBOL(empty_zero_page);
struct cachepolicy {
const char policy[16];
u64 mair;
u64 tcr;
};
static struct cachepolicy cache_policies[] __initdata = {
{
.policy = "uncached",
.mair = 0x44, /* inner, outer non-cacheable */
.tcr = TCR_IRGN_NC | TCR_ORGN_NC,
}, {
.policy = "writethrough",
.mair = 0xaa, /* inner, outer write-through, read-allocate */
.tcr = TCR_IRGN_WT | TCR_ORGN_WT,
}, {
.policy = "writeback",
.mair = 0xee, /* inner, outer write-back, read-allocate */
.tcr = TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
}
};
/*
* These are useful for identifying cache coherency problems by allowing the
* cache or the cache and writebuffer to be turned off. It changes the Normal
* memory caching attributes in the MAIR_EL1 register.
*/
static int __init early_cachepolicy(char *p)
{
int i;
u64 tmp;
for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
int len = strlen(cache_policies[i].policy);
if (memcmp(p, cache_policies[i].policy, len) == 0)
break;
}
if (i == ARRAY_SIZE(cache_policies)) {
pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
return 0;
}
flush_cache_all();
/*
* Modify MT_NORMAL attributes in MAIR_EL1.
*/
asm volatile(
" mrs %0, mair_el1\n"
" bfi %0, %1, %2, #8\n"
" msr mair_el1, %0\n"
" isb\n"
: "=&r" (tmp)
: "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
/*
* Modify TCR PTW cacheability attributes.
*/
asm volatile(
" mrs %0, tcr_el1\n"
" bic %0, %0, %2\n"
" orr %0, %0, %1\n"
" msr tcr_el1, %0\n"
" isb\n"
: "=&r" (tmp)
: "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
flush_cache_all();
return 0;
}
early_param("cachepolicy", early_cachepolicy);
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
unsigned long size, pgprot_t vma_prot)
{
@ -133,19 +61,42 @@ EXPORT_SYMBOL(phys_mem_access_prot);
static void __init *early_alloc(unsigned long sz)
{
void *ptr = __va(memblock_alloc(sz, sz));
BUG_ON(!ptr);
memset(ptr, 0, sz);
return ptr;
}
static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
/*
* remap a PMD into pages
*/
static void split_pmd(pmd_t *pmd, pte_t *pte)
{
unsigned long pfn = pmd_pfn(*pmd);
int i = 0;
do {
/*
* Need to have the least restrictive permissions available
* permissions will be fixed up later
*/
set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
pfn++;
} while (pte++, i++, i < PTRS_PER_PTE);
}
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
unsigned long end, unsigned long pfn,
pgprot_t prot)
pgprot_t prot,
void *(*alloc)(unsigned long size))
{
pte_t *pte;
if (pmd_none(*pmd)) {
pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t));
if (pmd_none(*pmd) || pmd_sect(*pmd)) {
pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
if (pmd_sect(*pmd))
split_pmd(pmd, pte);
__pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
flush_tlb_all();
}
BUG_ON(pmd_bad(*pmd));
@ -156,30 +107,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
} while (pte++, addr += PAGE_SIZE, addr != end);
}
static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
unsigned long end, phys_addr_t phys,
int map_io)
void split_pud(pud_t *old_pud, pmd_t *pmd)
{
unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
int i = 0;
do {
set_pmd(pmd, __pmd(addr | prot));
addr += PMD_SIZE;
} while (pmd++, i++, i < PTRS_PER_PMD);
}
static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
void *(*alloc)(unsigned long size))
{
pmd_t *pmd;
unsigned long next;
pmdval_t prot_sect;
pgprot_t prot_pte;
if (map_io) {
prot_sect = PROT_SECT_DEVICE_nGnRE;
prot_pte = __pgprot(PROT_DEVICE_nGnRE);
} else {
prot_sect = PROT_SECT_NORMAL_EXEC;
prot_pte = PAGE_KERNEL_EXEC;
}
/*
* Check for initial section mappings in the pgd/pud and remove them.
*/
if (pud_none(*pud) || pud_bad(*pud)) {
pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
pud_populate(&init_mm, pud, pmd);
if (pud_none(*pud) || pud_sect(*pud)) {
pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t));
if (pud_sect(*pud)) {
/*
* need to have the 1G of mappings continue to be
* present
*/
split_pud(pud, pmd);
}
pud_populate(mm, pud, pmd);
flush_tlb_all();
}
BUG_ON(pud_bad(*pud));
pmd = pmd_offset(pud, addr);
do {
@ -187,31 +150,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
/* try section mapping first */
if (((addr | next | phys) & ~SECTION_MASK) == 0) {
pmd_t old_pmd =*pmd;
set_pmd(pmd, __pmd(phys | prot_sect));
set_pmd(pmd, __pmd(phys |
pgprot_val(mk_sect_prot(prot))));
/*
* Check for previous table entries created during
* boot (__create_page_tables) and flush them.
*/
if (!pmd_none(old_pmd))
if (!pmd_none(old_pmd)) {
flush_tlb_all();
if (pmd_table(old_pmd)) {
phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0));
if (!WARN_ON_ONCE(slab_is_available()))
memblock_free(table, PAGE_SIZE);
}
}
} else {
alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
prot_pte);
prot, alloc);
}
phys += next - addr;
} while (pmd++, addr = next, addr != end);
}
static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
unsigned long end, phys_addr_t phys,
int map_io)
static inline bool use_1G_block(unsigned long addr, unsigned long next,
unsigned long phys)
{
if (PAGE_SHIFT != 12)
return false;
if (((addr | next | phys) & ~PUD_MASK) != 0)
return false;
return true;
}
static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
void *(*alloc)(unsigned long size))
{
pud_t *pud;
unsigned long next;
if (pgd_none(*pgd)) {
pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
pgd_populate(&init_mm, pgd, pud);
pud = alloc(PTRS_PER_PUD * sizeof(pud_t));
pgd_populate(mm, pgd, pud);
}
BUG_ON(pgd_bad(*pgd));
@ -222,10 +205,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
/*
* For 4K granule only, attempt to put down a 1GB block
*/
if (!map_io && (PAGE_SHIFT == 12) &&
((addr | next | phys) & ~PUD_MASK) == 0) {
if (use_1G_block(addr, next, phys)) {
pud_t old_pud = *pud;
set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC));
set_pud(pud, __pud(phys |
pgprot_val(mk_sect_prot(prot))));
/*
* If we have an old value for a pud, it will
@ -235,12 +218,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
* Look up the old pmd table and free it.
*/
if (!pud_none(old_pud)) {
phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
memblock_free(table, PAGE_SIZE);
flush_tlb_all();
if (pud_table(old_pud)) {
phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
if (!WARN_ON_ONCE(slab_is_available()))
memblock_free(table, PAGE_SIZE);
}
}
} else {
alloc_init_pmd(pud, addr, next, phys, map_io);
alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc);
}
phys += next - addr;
} while (pud++, addr = next, addr != end);
@ -250,9 +236,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
* Create the page directory entries and any necessary page tables for the
* mapping specified by 'md'.
*/
static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
int map_io)
static void __create_mapping(struct mm_struct *mm, pgd_t *pgd,
phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot,
void *(*alloc)(unsigned long size))
{
unsigned long addr, length, end, next;
@ -262,32 +249,96 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
end = addr + length;
do {
next = pgd_addr_end(addr, end);
alloc_init_pud(pgd, addr, next, phys, map_io);
alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc);
phys += next - addr;
} while (pgd++, addr = next, addr != end);
}
static void __init create_mapping(phys_addr_t phys, unsigned long virt,
phys_addr_t size)
static void *late_alloc(unsigned long size)
{
void *ptr;
BUG_ON(size > PAGE_SIZE);
ptr = (void *)__get_free_page(PGALLOC_GFP);
BUG_ON(!ptr);
return ptr;
}
static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot)
{
if (virt < VMALLOC_START) {
pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
&phys, virt);
return;
}
__create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0);
__create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
size, prot, early_alloc);
}
void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io)
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot)
{
if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) {
pr_warn("BUG: not creating id mapping for %pa\n", &addr);
__create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot,
late_alloc);
}
static void create_mapping_late(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot)
{
if (virt < VMALLOC_START) {
pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
&phys, virt);
return;
}
__create_mapping(&idmap_pg_dir[pgd_index(addr)],
addr, addr, size, map_io);
return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK),
phys, virt, size, prot, late_alloc);
}
#ifdef CONFIG_DEBUG_RODATA
static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
{
/*
* Set up the executable regions using the existing section mappings
* for now. This will get more fine grained later once all memory
* is mapped
*/
unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
if (end < kernel_x_start) {
create_mapping(start, __phys_to_virt(start),
end - start, PAGE_KERNEL);
} else if (start >= kernel_x_end) {
create_mapping(start, __phys_to_virt(start),
end - start, PAGE_KERNEL);
} else {
if (start < kernel_x_start)
create_mapping(start, __phys_to_virt(start),
kernel_x_start - start,
PAGE_KERNEL);
create_mapping(kernel_x_start,
__phys_to_virt(kernel_x_start),
kernel_x_end - kernel_x_start,
PAGE_KERNEL_EXEC);
if (kernel_x_end < end)
create_mapping(kernel_x_end,
__phys_to_virt(kernel_x_end),
end - kernel_x_end,
PAGE_KERNEL);
}
}
#else
static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
{
create_mapping(start, __phys_to_virt(start), end - start,
PAGE_KERNEL_EXEC);
}
#endif
static void __init map_mem(void)
{
struct memblock_region *reg;
@ -332,14 +383,53 @@ static void __init map_mem(void)
memblock_set_current_limit(limit);
}
#endif
create_mapping(start, __phys_to_virt(start), end - start);
__map_memblock(start, end);
}
/* Limit no longer required. */
memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
}
void __init fixup_executable(void)
{
#ifdef CONFIG_DEBUG_RODATA
/* now that we are actually fully mapped, make the start/end more fine grained */
if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) {
unsigned long aligned_start = round_down(__pa(_stext),
SECTION_SIZE);
create_mapping(aligned_start, __phys_to_virt(aligned_start),
__pa(_stext) - aligned_start,
PAGE_KERNEL);
}
if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) {
unsigned long aligned_end = round_up(__pa(__init_end),
SECTION_SIZE);
create_mapping(__pa(__init_end), (unsigned long)__init_end,
aligned_end - __pa(__init_end),
PAGE_KERNEL);
}
#endif
}
#ifdef CONFIG_DEBUG_RODATA
void mark_rodata_ro(void)
{
create_mapping_late(__pa(_stext), (unsigned long)_stext,
(unsigned long)_etext - (unsigned long)_stext,
PAGE_KERNEL_EXEC | PTE_RDONLY);
}
#endif
void fixup_init(void)
{
create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin,
(unsigned long)__init_end - (unsigned long)__init_begin,
PAGE_KERNEL);
}
/*
* paging_init() sets up the page tables, initialises the zone memory
* maps and sets up the zero page.
@ -349,13 +439,7 @@ void __init paging_init(void)
void *zero_page;
map_mem();
/*
* Finally flush the caches and tlb to ensure that we're in a
* consistent state.
*/
flush_cache_all();
flush_tlb_all();
fixup_executable();
/* allocate the zero page. */
zero_page = early_alloc(PAGE_SIZE);

14
arch/arm64/mm/proc.S
View File

@ -102,7 +102,7 @@ ENTRY(cpu_do_idle)
ret
ENDPROC(cpu_do_idle)
#ifdef CONFIG_ARM64_CPU_SUSPEND
#ifdef CONFIG_CPU_PM
/**
* cpu_do_suspend - save CPU registers context
*
@ -244,14 +244,18 @@ ENTRY(__cpu_setup)
ENDPROC(__cpu_setup)
/*
* We set the desired value explicitly, including those of the
* reserved bits. The values of bits EE & E0E were set early in
* el2_setup, which are left untouched below.
*
* n n T
* U E WT T UD US IHBS
* CE0 XWHW CZ ME TEEA S
* .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
* 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved
* .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings
* 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
* .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
*/
.type crval, #object
crval:
.word 0x000802e2 // clear
.word 0x0405d11d // set
.word 0xfcffffff // clear
.word 0x34d5d91d // set

1
drivers/cpuidle/Kconfig.arm64
View File

@ -4,7 +4,6 @@
config ARM64_CPUIDLE
bool "Generic ARM64 CPU idle Driver"
select ARM64_CPU_SUSPEND
select DT_IDLE_STATES
help
Select this to enable generic cpuidle driver for ARM64.

1
drivers/cpuidle/cpuidle-arm64.c
View File

@ -19,7 +19,6 @@
#include <linux/of.h>
#include <asm/cpuidle.h>
#include <asm/suspend.h>
#include "dt_idle_states.h"

80
drivers/firmware/efi/efi.c
View File

@ -297,10 +297,49 @@ static __init int match_config_table(efi_guid_t *guid,
return 0;
}
int __init efi_config_parse_tables(void *config_tables, int count, int sz,
efi_config_table_type_t *arch_tables)
{
void *tablep;
int i;
tablep = config_tables;
pr_info("");
for (i = 0; i < count; i++) {
efi_guid_t guid;
unsigned long table;
if (efi_enabled(EFI_64BIT)) {
u64 table64;
guid = ((efi_config_table_64_t *)tablep)->guid;
table64 = ((efi_config_table_64_t *)tablep)->table;
table = table64;
#ifndef CONFIG_64BIT
if (table64 >> 32) {
pr_cont("\n");
pr_err("Table located above 4GB, disabling EFI.\n");
return -EINVAL;
}
#endif
} else {
guid = ((efi_config_table_32_t *)tablep)->guid;
table = ((efi_config_table_32_t *)tablep)->table;
}
if (!match_config_table(&guid, table, common_tables))
match_config_table(&guid, table, arch_tables);
tablep += sz;
}
pr_cont("\n");
set_bit(EFI_CONFIG_TABLES, &efi.flags);
return 0;
}
int __init efi_config_init(efi_config_table_type_t *arch_tables)
{
void *config_tables, *tablep;
int i, sz;
void *config_tables;
int sz, ret;
if (efi_enabled(EFI_64BIT))
sz = sizeof(efi_config_table_64_t);
@ -317,42 +356,11 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables)
return -ENOMEM;
}
tablep = config_tables;
pr_info("");
for (i = 0; i < efi.systab->nr_tables; i++) {
efi_guid_t guid;
unsigned long table;
ret = efi_config_parse_tables(config_tables, efi.systab->nr_tables, sz,
arch_tables);
if (efi_enabled(EFI_64BIT)) {
u64 table64;
guid = ((efi_config_table_64_t *)tablep)->guid;
table64 = ((efi_config_table_64_t *)tablep)->table;
table = table64;
#ifndef CONFIG_64BIT
if (table64 >> 32) {
pr_cont("\n");
pr_err("Table located above 4GB, disabling EFI.\n");
early_memunmap(config_tables,
efi.systab->nr_tables * sz);
return -EINVAL;
}
#endif
} else {
guid = ((efi_config_table_32_t *)tablep)->guid;
table = ((efi_config_table_32_t *)tablep)->table;
}
if (!match_config_table(&guid, table, common_tables))
match_config_table(&guid, table, arch_tables);
tablep += sz;
}
pr_cont("\n");
early_memunmap(config_tables, efi.systab->nr_tables * sz);
set_bit(EFI_CONFIG_TABLES, &efi.flags);
return 0;
return ret;
}
#ifdef CONFIG_EFI_VARS_MODULE

59
drivers/firmware/efi/libstub/arm-stub.c
View File

@ -295,3 +295,62 @@ fail_free_image:
fail:
return EFI_ERROR;
}
/*
* This is the base address at which to start allocating virtual memory ranges
* for UEFI Runtime Services. This is in the low TTBR0 range so that we can use
* any allocation we choose, and eliminate the risk of a conflict after kexec.
* The value chosen is the largest non-zero power of 2 suitable for this purpose
* both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can
* be mapped efficiently.
*/
#define EFI_RT_VIRTUAL_BASE 0x40000000
/*
* efi_get_virtmap() - create a virtual mapping for the EFI memory map
*
* This function populates the virt_addr fields of all memory region descriptors
* in @memory_map whose EFI_MEMORY_RUNTIME attribute is set. Those descriptors
* are also copied to @runtime_map, and their total count is returned in @count.
*/
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
unsigned long desc_size, efi_memory_desc_t *runtime_map,
int *count)
{
u64 efi_virt_base = EFI_RT_VIRTUAL_BASE;
efi_memory_desc_t *out = runtime_map;
int l;
for (l = 0; l < map_size; l += desc_size) {
efi_memory_desc_t *in = (void *)memory_map + l;
u64 paddr, size;
if (!(in->attribute & EFI_MEMORY_RUNTIME))
continue;
/*
* Make the mapping compatible with 64k pages: this allows
* a 4k page size kernel to kexec a 64k page size kernel and
* vice versa.
*/
paddr = round_down(in->phys_addr, SZ_64K);
size = round_up(in->num_pages * EFI_PAGE_SIZE +
in->phys_addr - paddr, SZ_64K);
/*
* Avoid wasting memory on PTEs by choosing a virtual base that
* is compatible with section mappings if this region has the
* appropriate size and physical alignment. (Sections are 2 MB
* on 4k granule kernels)
*/
if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
efi_virt_base = round_up(efi_virt_base, SZ_2M);
in->virt_addr = efi_virt_base + in->phys_addr - paddr;
efi_virt_base += size;
memcpy(out, in, desc_size);
out = (void *)out + desc_size;
++*count;
}
}

25
drivers/firmware/efi/libstub/efi-stub-helper.c
View File

@ -32,6 +32,15 @@
static unsigned long __chunk_size = EFI_READ_CHUNK_SIZE;
/*
* Allow the platform to override the allocation granularity: this allows
* systems that have the capability to run with a larger page size to deal
* with the allocations for initrd and fdt more efficiently.
*/
#ifndef EFI_ALLOC_ALIGN
#define EFI_ALLOC_ALIGN EFI_PAGE_SIZE
#endif
struct file_info {
efi_file_handle_t *handle;
u64 size;
@ -154,10 +163,10 @@ efi_status_t efi_high_alloc(efi_system_table_t *sys_table_arg,
* a specific address. We are doing page-based allocations,
* so we must be aligned to a page.
*/
if (align < EFI_PAGE_SIZE)
align = EFI_PAGE_SIZE;
if (align < EFI_ALLOC_ALIGN)
align = EFI_ALLOC_ALIGN;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
again:
for (i = 0; i < map_size / desc_size; i++) {
efi_memory_desc_t *desc;
@ -239,10 +248,10 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg,
* a specific address. We are doing page-based allocations,
* so we must be aligned to a page.
*/
if (align < EFI_PAGE_SIZE)
align = EFI_PAGE_SIZE;
if (align < EFI_ALLOC_ALIGN)
align = EFI_ALLOC_ALIGN;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
for (i = 0; i < map_size / desc_size; i++) {
efi_memory_desc_t *desc;
unsigned long m = (unsigned long)map;
@ -296,7 +305,7 @@ void efi_free(efi_system_table_t *sys_table_arg, unsigned long size,
if (!size)
return;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
efi_call_early(free_pages, addr, nr_pages);
}
@ -565,7 +574,7 @@ efi_status_t efi_relocate_kernel(efi_system_table_t *sys_table_arg,
* to the preferred address. If that fails, allocate as low
* as possible while respecting the required alignment.
*/
nr_pages = round_up(alloc_size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(alloc_size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
status = efi_call_early(allocate_pages,
EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA,
nr_pages, &efi_addr);

4
drivers/firmware/efi/libstub/efistub.h
View File

@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
void *get_fdt(efi_system_table_t *sys_table);
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
unsigned long desc_size, efi_memory_desc_t *runtime_map,
int *count);
#endif

62
drivers/firmware/efi/libstub/fdt.c
View File

@ -14,6 +14,8 @@
#include <linux/libfdt.h>
#include <asm/efi.h>
#include "efistub.h"
efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
unsigned long orig_fdt_size,
void *fdt, int new_fdt_size, char *cmdline_ptr,
@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
unsigned long map_size, desc_size;
u32 desc_ver;
unsigned long mmap_key;
efi_memory_desc_t *memory_map;
efi_memory_desc_t *memory_map, *runtime_map;
unsigned long new_fdt_size;
efi_status_t status;
int runtime_entry_count = 0;
/*
* Get a copy of the current memory map that we will use to prepare
* the input for SetVirtualAddressMap(). We don't have to worry about
* subsequent allocations adding entries, since they could not affect
* the number of EFI_MEMORY_RUNTIME regions.
*/
status = efi_get_memory_map(sys_table, &runtime_map, &map_size,
&desc_size, &desc_ver, &mmap_key);
if (status != EFI_SUCCESS) {
pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n");
return status;
}
pr_efi(sys_table,
"Exiting boot services and installing virtual address map...\n");
/*
* Estimate size of new FDT, and allocate memory for it. We
@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
}
}
/*
* Update the memory map with virtual addresses. The function will also
* populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
* entries so that we can pass it straight into SetVirtualAddressMap()
*/
efi_get_virtmap(memory_map, map_size, desc_size, runtime_map,
&runtime_entry_count);
/* Now we are ready to exit_boot_services.*/
status = sys_table->boottime->exit_boot_services(handle, mmap_key);
if (status == EFI_SUCCESS) {
efi_set_virtual_address_map_t *svam;
if (status == EFI_SUCCESS)
return status;
/* Install the new virtual address map */
svam = sys_table->runtime->set_virtual_address_map;
status = svam(runtime_entry_count * desc_size, desc_size,
desc_ver, runtime_map);
/*
* We are beyond the point of no return here, so if the call to
* SetVirtualAddressMap() failed, we need to signal that to the
* incoming kernel but proceed normally otherwise.
*/
if (status != EFI_SUCCESS) {
int l;
/*
* Set the virtual address field of all
* EFI_MEMORY_RUNTIME entries to 0. This will signal
* the incoming kernel that no virtual translation has
* been installed.
*/
for (l = 0; l < map_size; l += desc_size) {
efi_memory_desc_t *p = (void *)memory_map + l;
if (p->attribute & EFI_MEMORY_RUNTIME)
p->virt_addr = 0;
}
}
return EFI_SUCCESS;
}
pr_efi_err(sys_table, "Exit boot services failed.\n");
@ -264,6 +319,7 @@ fail_free_new_fdt:
efi_free(sys_table, new_fdt_size, *new_fdt_addr);
fail:
sys_table->boottime->free_pool(runtime_map);
return EFI_LOAD_ERROR;
}

9
include/linux/compat.h
View File

@ -689,6 +689,15 @@ asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd,
asmlinkage long compat_sys_sigaltstack(const compat_stack_t __user *uss_ptr,
compat_stack_t __user *uoss_ptr);
#ifdef __ARCH_WANT_SYS_SIGPENDING
asmlinkage long compat_sys_sigpending(compat_old_sigset_t __user *set);
#endif
#ifdef __ARCH_WANT_SYS_SIGPROCMASK
asmlinkage long compat_sys_sigprocmask(int how, compat_old_sigset_t __user *nset,
compat_old_sigset_t __user *oset);
#endif
int compat_restore_altstack(const compat_stack_t __user *uss);
int __compat_save_altstack(compat_stack_t __user *, unsigned long);
#define compat_save_altstack_ex(uss, sp) do { \

2
include/linux/efi.h
View File

@ -875,6 +875,8 @@ static inline efi_status_t efi_query_variable_store(u32 attributes, unsigned lon
#endif
extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr);
extern int efi_config_init(efi_config_table_type_t *arch_tables);
extern int efi_config_parse_tables(void *config_tables, int count, int sz,
efi_config_table_type_t *arch_tables);
extern u64 efi_get_iobase (void);
extern u32 efi_mem_type (unsigned long phys_addr);
extern u64 efi_mem_attributes (unsigned long phys_addr);

8
include/linux/syscalls.h
View File

@ -410,12 +410,16 @@ asmlinkage long sys_newlstat(const char __user *filename,
struct stat __user *statbuf);
asmlinkage long sys_newfstat(unsigned int fd, struct stat __user *statbuf);
asmlinkage long sys_ustat(unsigned dev, struct ustat __user *ubuf);
#if BITS_PER_LONG == 32
#if defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_COMPAT_STAT64)
asmlinkage long sys_stat64(const char __user *filename,
struct stat64 __user *statbuf);
asmlinkage long sys_fstat64(unsigned long fd, struct stat64 __user *statbuf);
asmlinkage long sys_lstat64(const char __user *filename,
struct stat64 __user *statbuf);
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
struct stat64 __user *statbuf, int flag);
#endif
#if BITS_PER_LONG == 32
asmlinkage long sys_truncate64(const char __user *path, loff_t length);
asmlinkage long sys_ftruncate64(unsigned int fd, loff_t length);
#endif
@ -771,8 +775,6 @@ asmlinkage long sys_openat(int dfd, const char __user *filename, int flags,
umode_t mode);
asmlinkage long sys_newfstatat(int dfd, const char __user *filename,
struct stat __user *statbuf, int flag);
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
struct stat64 __user *statbuf, int flag);
asmlinkage long sys_readlinkat(int dfd, const char __user *path, char __user *buf,
int bufsiz);
asmlinkage long sys_utimensat(int dfd, const char __user *filename,