2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
* Based on arch/arm/mm/fault.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
|
|
* Copyright (C) 1995-2004 Russell King
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/signal.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/kprobes.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/page-flags.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/perf_event.h>
|
|
|
|
|
2015-07-22 18:05:54 +00:00
|
|
|
#include <asm/cpufeature.h>
|
2012-03-05 11:49:27 +00:00
|
|
|
#include <asm/exception.h>
|
|
|
|
#include <asm/debug-monitors.h>
|
2014-04-06 22:04:12 +00:00
|
|
|
#include <asm/esr.h>
|
2015-07-22 18:05:54 +00:00
|
|
|
#include <asm/sysreg.h>
|
2012-03-05 11:49:27 +00:00
|
|
|
#include <asm/system_misc.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2012-10-24 15:34:02 +00:00
|
|
|
static const char *fault_name(unsigned int esr);
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
* Dump out the page tables associated with 'addr' in mm 'mm'.
|
|
|
|
*/
|
|
|
|
void show_pte(struct mm_struct *mm, unsigned long addr)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
|
|
if (!mm)
|
|
|
|
mm = &init_mm;
|
|
|
|
|
|
|
|
pr_alert("pgd = %p\n", mm->pgd);
|
|
|
|
pgd = pgd_offset(mm, addr);
|
|
|
|
pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd));
|
|
|
|
|
|
|
|
do {
|
|
|
|
pud_t *pud;
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
|
|
|
|
2013-04-19 14:49:31 +00:00
|
|
|
if (pgd_none(*pgd) || pgd_bad(*pgd))
|
2012-03-05 11:49:27 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
pud = pud_offset(pgd, addr);
|
2014-05-12 09:40:51 +00:00
|
|
|
printk(", *pud=%016llx", pud_val(*pud));
|
2013-04-19 14:49:31 +00:00
|
|
|
if (pud_none(*pud) || pud_bad(*pud))
|
2012-03-05 11:49:27 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
pmd = pmd_offset(pud, addr);
|
|
|
|
printk(", *pmd=%016llx", pmd_val(*pmd));
|
2013-04-19 14:49:31 +00:00
|
|
|
if (pmd_none(*pmd) || pmd_bad(*pmd))
|
2012-03-05 11:49:27 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
pte = pte_offset_map(pmd, addr);
|
|
|
|
printk(", *pte=%016llx", pte_val(*pte));
|
|
|
|
pte_unmap(pte);
|
|
|
|
} while(0);
|
|
|
|
|
|
|
|
printk("\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The kernel tried to access some page that wasn't present.
|
|
|
|
*/
|
|
|
|
static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
|
|
|
|
unsigned int esr, struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Are we prepared to handle this kernel fault?
|
|
|
|
*/
|
|
|
|
if (fixup_exception(regs))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No handler, we'll have to terminate things with extreme prejudice.
|
|
|
|
*/
|
|
|
|
bust_spinlocks(1);
|
|
|
|
pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
|
|
|
|
(addr < PAGE_SIZE) ? "NULL pointer dereference" :
|
|
|
|
"paging request", addr);
|
|
|
|
|
|
|
|
show_pte(mm, addr);
|
|
|
|
die("Oops", regs, esr);
|
|
|
|
bust_spinlocks(0);
|
|
|
|
do_exit(SIGKILL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Something tried to access memory that isn't in our memory map. User mode
|
|
|
|
* accesses just cause a SIGSEGV
|
|
|
|
*/
|
|
|
|
static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
|
|
|
|
unsigned int esr, unsigned int sig, int code,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct siginfo si;
|
|
|
|
|
2015-07-03 14:08:08 +00:00
|
|
|
if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) {
|
2012-10-24 15:34:02 +00:00
|
|
|
pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n",
|
|
|
|
tsk->comm, task_pid_nr(tsk), fault_name(esr), sig,
|
|
|
|
addr, esr);
|
2012-03-05 11:49:27 +00:00
|
|
|
show_pte(tsk->mm, addr);
|
|
|
|
show_regs(regs);
|
|
|
|
}
|
|
|
|
|
|
|
|
tsk->thread.fault_address = addr;
|
2014-04-06 22:04:12 +00:00
|
|
|
tsk->thread.fault_code = esr;
|
2012-03-05 11:49:27 +00:00
|
|
|
si.si_signo = sig;
|
|
|
|
si.si_errno = 0;
|
|
|
|
si.si_code = code;
|
|
|
|
si.si_addr = (void __user *)addr;
|
|
|
|
force_sig_info(sig, &si, tsk);
|
|
|
|
}
|
|
|
|
|
2013-09-16 14:18:28 +00:00
|
|
|
static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
2012-03-05 11:49:27 +00:00
|
|
|
{
|
|
|
|
struct task_struct *tsk = current;
|
|
|
|
struct mm_struct *mm = tsk->active_mm;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we are in kernel mode at this point, we have no context to
|
|
|
|
* handle this fault with.
|
|
|
|
*/
|
|
|
|
if (user_mode(regs))
|
|
|
|
__do_user_fault(tsk, addr, esr, SIGSEGV, SEGV_MAPERR, regs);
|
|
|
|
else
|
|
|
|
__do_kernel_fault(mm, addr, esr, regs);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define VM_FAULT_BADMAP 0x010000
|
|
|
|
#define VM_FAULT_BADACCESS 0x020000
|
|
|
|
|
|
|
|
#define ESR_LNX_EXEC (1 << 24)
|
|
|
|
|
|
|
|
static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
|
2013-07-19 14:37:12 +00:00
|
|
|
unsigned int mm_flags, unsigned long vm_flags,
|
2012-03-05 11:49:27 +00:00
|
|
|
struct task_struct *tsk)
|
|
|
|
{
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
int fault;
|
|
|
|
|
|
|
|
vma = find_vma(mm, addr);
|
|
|
|
fault = VM_FAULT_BADMAP;
|
|
|
|
if (unlikely(!vma))
|
|
|
|
goto out;
|
|
|
|
if (unlikely(vma->vm_start > addr))
|
|
|
|
goto check_stack;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ok, we have a good vm_area for this memory access, so we can handle
|
|
|
|
* it.
|
|
|
|
*/
|
|
|
|
good_area:
|
2013-07-19 14:37:12 +00:00
|
|
|
/*
|
|
|
|
* Check that the permissions on the VMA allow for the fault which
|
2014-05-16 15:44:32 +00:00
|
|
|
* occurred. If we encountered a write or exec fault, we must have
|
|
|
|
* appropriate permissions, otherwise we allow any permission.
|
2013-07-19 14:37:12 +00:00
|
|
|
*/
|
|
|
|
if (!(vma->vm_flags & vm_flags)) {
|
2012-03-05 11:49:27 +00:00
|
|
|
fault = VM_FAULT_BADACCESS;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2013-07-19 14:37:12 +00:00
|
|
|
return handle_mm_fault(mm, vma, addr & PAGE_MASK, mm_flags);
|
2012-03-05 11:49:27 +00:00
|
|
|
|
|
|
|
check_stack:
|
|
|
|
if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
|
|
|
|
goto good_area;
|
|
|
|
out:
|
|
|
|
return fault;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct task_struct *tsk;
|
|
|
|
struct mm_struct *mm;
|
|
|
|
int fault, sig, code;
|
2014-05-16 15:44:32 +00:00
|
|
|
unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
|
2013-07-19 14:37:12 +00:00
|
|
|
unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
tsk = current;
|
|
|
|
mm = tsk->mm;
|
|
|
|
|
|
|
|
/* Enable interrupts if they were enabled in the parent context. */
|
|
|
|
if (interrupts_enabled(regs))
|
|
|
|
local_irq_enable();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we're in an interrupt or have no user context, we must not take
|
|
|
|
* the fault.
|
|
|
|
*/
|
2015-05-11 15:52:11 +00:00
|
|
|
if (faulthandler_disabled() || !mm)
|
2012-03-05 11:49:27 +00:00
|
|
|
goto no_context;
|
|
|
|
|
2013-09-12 22:13:39 +00:00
|
|
|
if (user_mode(regs))
|
|
|
|
mm_flags |= FAULT_FLAG_USER;
|
|
|
|
|
|
|
|
if (esr & ESR_LNX_EXEC) {
|
|
|
|
vm_flags = VM_EXEC;
|
2014-11-24 12:31:40 +00:00
|
|
|
} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
|
2013-09-12 22:13:39 +00:00
|
|
|
vm_flags = VM_WRITE;
|
|
|
|
mm_flags |= FAULT_FLAG_WRITE;
|
|
|
|
}
|
|
|
|
|
2015-07-22 18:05:54 +00:00
|
|
|
/*
|
|
|
|
* PAN bit set implies the fault happened in kernel space, but not
|
|
|
|
* in the arch's user access functions.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT))
|
|
|
|
goto no_context;
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
* As per x86, we may deadlock here. However, since the kernel only
|
|
|
|
* validly references user space from well defined areas of the code,
|
|
|
|
* we can bug out early if this is from code which shouldn't.
|
|
|
|
*/
|
|
|
|
if (!down_read_trylock(&mm->mmap_sem)) {
|
|
|
|
if (!user_mode(regs) && !search_exception_tables(regs->pc))
|
|
|
|
goto no_context;
|
|
|
|
retry:
|
|
|
|
down_read(&mm->mmap_sem);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The above down_read_trylock() might have succeeded in which
|
|
|
|
* case, we'll have missed the might_sleep() from down_read().
|
|
|
|
*/
|
|
|
|
might_sleep();
|
|
|
|
#ifdef CONFIG_DEBUG_VM
|
|
|
|
if (!user_mode(regs) && !search_exception_tables(regs->pc))
|
|
|
|
goto no_context;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-07-19 14:37:12 +00:00
|
|
|
fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
|
2012-03-05 11:49:27 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we need to retry but a fatal signal is pending, handle the
|
|
|
|
* signal first. We do not need to release the mmap_sem because it
|
|
|
|
* would already be released in __lock_page_or_retry in mm/filemap.c.
|
|
|
|
*/
|
|
|
|
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Major/minor page fault accounting is only done on the initial
|
|
|
|
* attempt. If we go through a retry, it is extremely likely that the
|
|
|
|
* page will be found in page cache at that point.
|
|
|
|
*/
|
|
|
|
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
|
2013-07-19 14:37:12 +00:00
|
|
|
if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
|
2012-03-05 11:49:27 +00:00
|
|
|
if (fault & VM_FAULT_MAJOR) {
|
|
|
|
tsk->maj_flt++;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
|
|
|
|
addr);
|
|
|
|
} else {
|
|
|
|
tsk->min_flt++;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
|
|
|
|
addr);
|
|
|
|
}
|
|
|
|
if (fault & VM_FAULT_RETRY) {
|
|
|
|
/*
|
|
|
|
* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
|
|
|
|
* starvation.
|
|
|
|
*/
|
2013-07-19 14:37:12 +00:00
|
|
|
mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
|
arm64: readahead: fault retry breaks mmap file read random detection
This is the arm64 portion of commit 45cac65b0fcd ("readahead: fault
retry breaks mmap file read random detection"), which was absent from
the initial port and has since gone unnoticed. The original commit says:
> .fault now can retry. The retry can break state machine of .fault. In
> filemap_fault, if page is miss, ra->mmap_miss is increased. In the second
> try, since the page is in page cache now, ra->mmap_miss is decreased. And
> these are done in one fault, so we can't detect random mmap file access.
>
> Add a new flag to indicate .fault is tried once. In the second try, skip
> ra->mmap_miss decreasing. The filemap_fault state machine is ok with it.
With this change, Mark reports that:
> Random read improves by 250%, sequential read improves by 40%, and
> random write by 400% to an eMMC device with dm crypto wrapped around it.
Cc: Shaohua Li <shli@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Signed-off-by: Riley Andrews <riandrews@android.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2015-09-21 20:39:50 +00:00
|
|
|
mm_flags |= FAULT_FLAG_TRIED;
|
2012-03-05 11:49:27 +00:00
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
up_read(&mm->mmap_sem);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
|
|
|
|
*/
|
|
|
|
if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
|
|
|
|
VM_FAULT_BADACCESS))))
|
|
|
|
return 0;
|
|
|
|
|
2013-09-12 22:13:38 +00:00
|
|
|
/*
|
|
|
|
* If we are in kernel mode at this point, we have no context to
|
|
|
|
* handle this fault with.
|
|
|
|
*/
|
|
|
|
if (!user_mode(regs))
|
|
|
|
goto no_context;
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
if (fault & VM_FAULT_OOM) {
|
|
|
|
/*
|
|
|
|
* We ran out of memory, call the OOM killer, and return to
|
|
|
|
* userspace (which will retry the fault, or kill us if we got
|
|
|
|
* oom-killed).
|
|
|
|
*/
|
|
|
|
pagefault_out_of_memory();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fault & VM_FAULT_SIGBUS) {
|
|
|
|
/*
|
|
|
|
* We had some memory, but were unable to successfully fix up
|
|
|
|
* this page fault.
|
|
|
|
*/
|
|
|
|
sig = SIGBUS;
|
|
|
|
code = BUS_ADRERR;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Something tried to access memory that isn't in our memory
|
|
|
|
* map.
|
|
|
|
*/
|
|
|
|
sig = SIGSEGV;
|
|
|
|
code = fault == VM_FAULT_BADACCESS ?
|
|
|
|
SEGV_ACCERR : SEGV_MAPERR;
|
|
|
|
}
|
|
|
|
|
|
|
|
__do_user_fault(tsk, addr, esr, sig, code, regs);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
no_context:
|
|
|
|
__do_kernel_fault(mm, addr, esr, regs);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First Level Translation Fault Handler
|
|
|
|
*
|
|
|
|
* We enter here because the first level page table doesn't contain a valid
|
|
|
|
* entry for the address.
|
|
|
|
*
|
|
|
|
* If the address is in kernel space (>= TASK_SIZE), then we are probably
|
|
|
|
* faulting in the vmalloc() area.
|
|
|
|
*
|
|
|
|
* If the init_task's first level page tables contains the relevant entry, we
|
|
|
|
* copy the it to this task. If not, we send the process a signal, fixup the
|
|
|
|
* exception, or oops the kernel.
|
|
|
|
*
|
|
|
|
* NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
|
|
|
|
* or a critical region, and should only copy the information from the master
|
|
|
|
* page table, nothing more.
|
|
|
|
*/
|
|
|
|
static int __kprobes do_translation_fault(unsigned long addr,
|
|
|
|
unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
if (addr < TASK_SIZE)
|
|
|
|
return do_page_fault(addr, esr, regs);
|
|
|
|
|
|
|
|
do_bad_area(addr, esr, regs);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This abort handler always returns "fault".
|
|
|
|
*/
|
|
|
|
static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct fault_info {
|
|
|
|
int (*fn)(unsigned long addr, unsigned int esr, struct pt_regs *regs);
|
|
|
|
int sig;
|
|
|
|
int code;
|
|
|
|
const char *name;
|
|
|
|
} fault_info[] = {
|
|
|
|
{ do_bad, SIGBUS, 0, "ttbr address size fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "level 1 address size fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "level 2 address size fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "level 3 address size fault" },
|
2014-11-21 14:22:22 +00:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
|
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 8" },
|
2013-04-10 12:48:00 +00:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 12" },
|
2013-04-10 12:48:00 +00:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous external abort" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 17" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 18" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 19" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous parity error" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 25" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 26" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 27" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
|
|
|
|
{ do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 32" },
|
|
|
|
{ do_bad, SIGBUS, BUS_ADRALN, "alignment fault" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 34" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 35" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 36" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 37" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 38" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 39" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 40" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 41" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 42" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 43" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 44" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 45" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 46" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 47" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "TLB conflict abort" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 49" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 50" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 51" },
|
|
|
|
{ do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 54" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 55" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 56" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 57" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 58" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 59" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 60" },
|
2015-11-23 15:09:36 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "section domain fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "page domain fault" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 63" },
|
|
|
|
};
|
|
|
|
|
2012-10-24 15:34:02 +00:00
|
|
|
static const char *fault_name(unsigned int esr)
|
|
|
|
{
|
|
|
|
const struct fault_info *inf = fault_info + (esr & 63);
|
|
|
|
return inf->name;
|
|
|
|
}
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
* Dispatch a data abort to the relevant handler.
|
|
|
|
*/
|
|
|
|
asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
const struct fault_info *inf = fault_info + (esr & 63);
|
|
|
|
struct siginfo info;
|
|
|
|
|
|
|
|
if (!inf->fn(addr, esr, regs))
|
|
|
|
return;
|
|
|
|
|
|
|
|
pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
|
|
|
|
inf->name, esr, addr);
|
|
|
|
|
|
|
|
info.si_signo = inf->sig;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = inf->code;
|
|
|
|
info.si_addr = (void __user *)addr;
|
|
|
|
arm64_notify_die("", regs, &info, esr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Handle stack alignment exceptions.
|
|
|
|
*/
|
|
|
|
asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
|
|
|
|
unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct siginfo info;
|
2015-06-19 14:28:16 +00:00
|
|
|
struct task_struct *tsk = current;
|
|
|
|
|
|
|
|
if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS))
|
|
|
|
pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n",
|
|
|
|
tsk->comm, task_pid_nr(tsk),
|
|
|
|
esr_get_class_string(esr), (void *)regs->pc,
|
|
|
|
(void *)regs->sp);
|
2012-03-05 11:49:27 +00:00
|
|
|
|
|
|
|
info.si_signo = SIGBUS;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = BUS_ADRALN;
|
|
|
|
info.si_addr = (void __user *)addr;
|
2015-06-19 14:28:16 +00:00
|
|
|
arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr);
|
2012-03-05 11:49:27 +00:00
|
|
|
}
|
|
|
|
|
2015-07-24 15:37:48 +00:00
|
|
|
int __init early_brk64(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* __refdata because early_brk64 is __init, but the reference to it is
|
|
|
|
* clobbered at arch_initcall time.
|
|
|
|
* See traps.c and debug-monitors.c:debug_traps_init().
|
|
|
|
*/
|
|
|
|
static struct fault_info __refdata debug_fault_info[] = {
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
|
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
|
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 3" },
|
|
|
|
{ do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
|
|
|
|
{ do_bad, SIGTRAP, 0, "aarch32 vector catch" },
|
2015-07-24 15:37:48 +00:00
|
|
|
{ early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
|
2012-03-05 11:49:27 +00:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 7" },
|
|
|
|
};
|
|
|
|
|
|
|
|
void __init hook_debug_fault_code(int nr,
|
|
|
|
int (*fn)(unsigned long, unsigned int, struct pt_regs *),
|
|
|
|
int sig, int code, const char *name)
|
|
|
|
{
|
|
|
|
BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
|
|
|
|
|
|
|
|
debug_fault_info[nr].fn = fn;
|
|
|
|
debug_fault_info[nr].sig = sig;
|
|
|
|
debug_fault_info[nr].code = code;
|
|
|
|
debug_fault_info[nr].name = name;
|
|
|
|
}
|
|
|
|
|
|
|
|
asmlinkage int __exception do_debug_exception(unsigned long addr,
|
|
|
|
unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
|
|
|
|
struct siginfo info;
|
|
|
|
|
|
|
|
if (!inf->fn(addr, esr, regs))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
|
|
|
|
inf->name, esr, addr);
|
|
|
|
|
|
|
|
info.si_signo = inf->sig;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = inf->code;
|
|
|
|
info.si_addr = (void __user *)addr;
|
2014-04-06 22:04:12 +00:00
|
|
|
arm64_notify_die("", regs, &info, 0);
|
2012-03-05 11:49:27 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2015-07-22 18:05:54 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_ARM64_PAN
|
arm64: Delay cpu feature capability checks
At the moment we run through the arm64_features capability list for
each CPU and set the capability if one of the CPU supports it. This
could be problematic in a heterogeneous system with differing capabilities.
Delay the CPU feature checks until all the enabled CPUs are up(i.e,
smp_cpus_done(), so that we can make better decisions based on the
overall system capability. Once we decide and advertise the capabilities
the alternatives can be applied. From this state, we cannot roll back
a feature to disabled based on the values from a new hotplugged CPU,
due to the runtime patching and other reasons. So, for all new CPUs,
we need to make sure that they have the established system capabilities.
Failing which, we bring the CPU down, preventing it from turning online.
Once the capabilities are decided, any new CPU booting up goes through
verification to ensure that it has all the enabled capabilities and also
invokes the respective enable() method on the CPU.
The CPU errata checks are not delayed and is still executed per-CPU
to detect the respective capabilities. If we ever come across a non-errata
capability that needs to be checked on each-CPU, we could introduce them via
a new capability table(or introduce a flag), which can be processed per CPU.
The next patch will make the feature checks use the system wide
safe value of a feature register.
NOTE: The enable() methods associated with the capability is scheduled
on all the CPUs (which is the only use case at the moment). If we need
a different type of 'enable()' which only needs to be run once on any CPU,
we should be able to handle that when needed.
Signed-off-by: Suzuki K. Poulose <suzuki.poulose@arm.com>
Tested-by: Dave Martin <Dave.Martin@arm.com>
[catalin.marinas@arm.com: static variable and coding style fixes]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2015-10-19 13:24:50 +00:00
|
|
|
void cpu_enable_pan(void *__unused)
|
2015-07-22 18:05:54 +00:00
|
|
|
{
|
|
|
|
config_sctlr_el1(SCTLR_EL1_SPAN, 0);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_ARM64_PAN */
|