2019-06-03 05:44:50 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2012-03-05 11:49:29 +00:00
|
|
|
/*
|
|
|
|
|
* Based on arch/arm/kernel/irq.c
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1992 Linus Torvalds
|
|
|
|
|
* Modifications for ARM processor Copyright (C) 1995-2000 Russell King.
|
|
|
|
|
* Support for Dynamic Tick Timer Copyright (C) 2004-2005 Nokia Corporation.
|
|
|
|
|
* Dynamic Tick Timer written by Tony Lindgren <tony@atomide.com> and
|
|
|
|
|
* Tuukka Tikkanen <tuukka.tikkanen@elektrobit.com>.
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
|
2020-06-20 16:19:00 +00:00
|
|
|
#include <linux/hardirq.h>
|
2012-03-05 11:49:29 +00:00
|
|
|
#include <linux/init.h>
|
2022-11-17 12:09:02 +00:00
|
|
|
#include <linux/irq.h>
|
2013-01-14 12:39:31 +00:00
|
|
|
#include <linux/irqchip.h>
|
2019-06-11 09:38:09 +00:00
|
|
|
#include <linux/kprobes.h>
|
2022-11-17 12:09:02 +00:00
|
|
|
#include <linux/memory.h>
|
2020-11-30 23:34:42 +00:00
|
|
|
#include <linux/scs.h>
|
2012-03-05 11:49:29 +00:00
|
|
|
#include <linux/seq_file.h>
|
2022-11-17 12:09:02 +00:00
|
|
|
#include <linux/smp.h>
|
2017-07-21 13:25:33 +00:00
|
|
|
#include <linux/vmalloc.h>
|
2019-06-11 09:38:09 +00:00
|
|
|
#include <asm/daifflags.h>
|
2022-08-15 12:47:39 +00:00
|
|
|
#include <asm/exception.h>
|
2023-11-24 03:15:13 +00:00
|
|
|
#include <asm/numa.h>
|
2022-08-15 12:47:39 +00:00
|
|
|
#include <asm/softirq_stack.h>
|
2022-11-17 12:09:02 +00:00
|
|
|
#include <asm/stacktrace.h>
|
|
|
|
|
#include <asm/vmap_stack.h>
|
2012-03-05 11:49:29 +00:00
|
|
|
|
2019-01-31 14:58:39 +00:00
|
|
|
/* Only access this in an NMI enter/exit */
|
|
|
|
|
DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
|
|
|
|
|
|
2017-07-31 20:17:03 +00:00
|
|
|
DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
|
2015-12-04 11:02:26 +00:00
|
|
|
|
2020-11-30 23:34:42 +00:00
|
|
|
|
|
|
|
|
DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_SHADOW_CALL_STACK
|
|
|
|
|
DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static void init_irq_scs(void)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
2022-10-27 15:59:08 +00:00
|
|
|
if (!scs_is_enabled())
|
2020-11-30 23:34:42 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
|
per_cpu(irq_shadow_call_stack_ptr, cpu) =
|
2023-12-13 01:20:46 +00:00
|
|
|
scs_alloc(early_cpu_to_node(cpu));
|
2020-11-30 23:34:42 +00:00
|
|
|
}
|
|
|
|
|
|
2017-07-21 13:25:33 +00:00
|
|
|
#ifdef CONFIG_VMAP_STACK
|
2023-11-24 03:15:13 +00:00
|
|
|
static void __init init_irq_stacks(void)
|
2017-07-21 13:25:33 +00:00
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
unsigned long *p;
|
|
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
2023-11-24 03:15:13 +00:00
|
|
|
p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu));
|
2017-07-21 13:25:33 +00:00
|
|
|
per_cpu(irq_stack_ptr, cpu) = p;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
|
|
|
|
|
DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
|
|
|
|
|
|
2017-07-31 20:17:03 +00:00
|
|
|
static void init_irq_stacks(void)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
|
per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
|
|
|
|
|
}
|
2017-07-21 13:25:33 +00:00
|
|
|
#endif
|
2017-07-31 20:17:03 +00:00
|
|
|
|
2022-08-15 12:47:39 +00:00
|
|
|
#ifndef CONFIG_PREEMPT_RT
|
|
|
|
|
static void ____do_softirq(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
__do_softirq();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void do_softirq_own_stack(void)
|
|
|
|
|
{
|
|
|
|
|
call_on_irq_stack(NULL, ____do_softirq);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
arm64: irq: rework root IRQ handler registration
If we accidentally unmask IRQs before we've registered a root IRQ
handler, handle_arch_irq will be NULL, and the IRQ exception handler
will branch to a bogus address.
To make this easier to debug, this patch initialises handle_arch_irq to
a default handler which will panic(), making such problems easier to
debug. When we add support for FIQ handlers, we can follow the same
approach.
When we add support for a root FIQ handler, it's possible to have root
IRQ handler without an root FIQ handler, and in theory the inverse is
also possible. To permit this, and to keep the IRQ/FIQ registration
logic similar, this patch removes the panic in the absence of a root IRQ
controller. Instead, set_handle_irq() logs when a handler is registered,
which is sufficient for debug purposes.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-4-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:26 +00:00
|
|
|
static void default_handle_irq(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
panic("IRQ taken without a root IRQ handler\n");
|
|
|
|
|
}
|
|
|
|
|
|
arm64: irq: allow FIQs to be handled
On contemporary platforms we don't use FIQ, and treat any stray FIQ as a
fatal event. However, some platforms have an interrupt controller wired
to FIQ, and need to handle FIQ as part of regular operation.
So that we can support both cases dynamically, this patch updates the
FIQ exception handling code to operate the same way as the IRQ handling
code, with its own handle_arch_fiq handler. Where a root FIQ handler is
not registered, an unexpected FIQ exception will trigger the default FIQ
handler, which will panic() as today. Where a root FIQ handler is
registered, handling of the FIQ is deferred to that handler.
As el0_fiq_invalid_compat is supplanted by el0_fiq, the former is
removed. For !CONFIG_COMPAT builds we never expect to take an exception
from AArch32 EL0, so we keep the common el0_fiq_invalid handler.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-7-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:29 +00:00
|
|
|
static void default_handle_fiq(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
panic("FIQ taken without a root FIQ handler\n");
|
|
|
|
|
}
|
|
|
|
|
|
arm64: irq: rework root IRQ handler registration
If we accidentally unmask IRQs before we've registered a root IRQ
handler, handle_arch_irq will be NULL, and the IRQ exception handler
will branch to a bogus address.
To make this easier to debug, this patch initialises handle_arch_irq to
a default handler which will panic(), making such problems easier to
debug. When we add support for FIQ handlers, we can follow the same
approach.
When we add support for a root FIQ handler, it's possible to have root
IRQ handler without an root FIQ handler, and in theory the inverse is
also possible. To permit this, and to keep the IRQ/FIQ registration
logic similar, this patch removes the panic in the absence of a root IRQ
controller. Instead, set_handle_irq() logs when a handler is registered,
which is sufficient for debug purposes.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-4-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:26 +00:00
|
|
|
void (*handle_arch_irq)(struct pt_regs *) __ro_after_init = default_handle_irq;
|
arm64: irq: allow FIQs to be handled
On contemporary platforms we don't use FIQ, and treat any stray FIQ as a
fatal event. However, some platforms have an interrupt controller wired
to FIQ, and need to handle FIQ as part of regular operation.
So that we can support both cases dynamically, this patch updates the
FIQ exception handling code to operate the same way as the IRQ handling
code, with its own handle_arch_fiq handler. Where a root FIQ handler is
not registered, an unexpected FIQ exception will trigger the default FIQ
handler, which will panic() as today. Where a root FIQ handler is
registered, handling of the FIQ is deferred to that handler.
As el0_fiq_invalid_compat is supplanted by el0_fiq, the former is
removed. For !CONFIG_COMPAT builds we never expect to take an exception
from AArch32 EL0, so we keep the common el0_fiq_invalid handler.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-7-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:29 +00:00
|
|
|
void (*handle_arch_fiq)(struct pt_regs *) __ro_after_init = default_handle_fiq;
|
2021-03-15 11:56:25 +00:00
|
|
|
|
|
|
|
|
int __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
|
|
|
|
|
{
|
arm64: irq: rework root IRQ handler registration
If we accidentally unmask IRQs before we've registered a root IRQ
handler, handle_arch_irq will be NULL, and the IRQ exception handler
will branch to a bogus address.
To make this easier to debug, this patch initialises handle_arch_irq to
a default handler which will panic(), making such problems easier to
debug. When we add support for FIQ handlers, we can follow the same
approach.
When we add support for a root FIQ handler, it's possible to have root
IRQ handler without an root FIQ handler, and in theory the inverse is
also possible. To permit this, and to keep the IRQ/FIQ registration
logic similar, this patch removes the panic in the absence of a root IRQ
controller. Instead, set_handle_irq() logs when a handler is registered,
which is sufficient for debug purposes.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-4-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:26 +00:00
|
|
|
if (handle_arch_irq != default_handle_irq)
|
2021-03-15 11:56:25 +00:00
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
|
|
handle_arch_irq = handle_irq;
|
arm64: irq: rework root IRQ handler registration
If we accidentally unmask IRQs before we've registered a root IRQ
handler, handle_arch_irq will be NULL, and the IRQ exception handler
will branch to a bogus address.
To make this easier to debug, this patch initialises handle_arch_irq to
a default handler which will panic(), making such problems easier to
debug. When we add support for FIQ handlers, we can follow the same
approach.
When we add support for a root FIQ handler, it's possible to have root
IRQ handler without an root FIQ handler, and in theory the inverse is
also possible. To permit this, and to keep the IRQ/FIQ registration
logic similar, this patch removes the panic in the absence of a root IRQ
controller. Instead, set_handle_irq() logs when a handler is registered,
which is sufficient for debug purposes.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-4-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:26 +00:00
|
|
|
pr_info("Root IRQ handler: %ps\n", handle_irq);
|
2021-03-15 11:56:25 +00:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: irq: allow FIQs to be handled
On contemporary platforms we don't use FIQ, and treat any stray FIQ as a
fatal event. However, some platforms have an interrupt controller wired
to FIQ, and need to handle FIQ as part of regular operation.
So that we can support both cases dynamically, this patch updates the
FIQ exception handling code to operate the same way as the IRQ handling
code, with its own handle_arch_fiq handler. Where a root FIQ handler is
not registered, an unexpected FIQ exception will trigger the default FIQ
handler, which will panic() as today. Where a root FIQ handler is
registered, handling of the FIQ is deferred to that handler.
As el0_fiq_invalid_compat is supplanted by el0_fiq, the former is
removed. For !CONFIG_COMPAT builds we never expect to take an exception
from AArch32 EL0, so we keep the common el0_fiq_invalid handler.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Hector Martin <marcan@marcan.st>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210315115629.57191-7-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-03-15 11:56:29 +00:00
|
|
|
int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *))
|
|
|
|
|
{
|
|
|
|
|
if (handle_arch_fiq != default_handle_fiq)
|
|
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
|
|
handle_arch_fiq = handle_fiq;
|
|
|
|
|
pr_info("Root FIQ handler: %ps\n", handle_fiq);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2012-03-05 11:49:29 +00:00
|
|
|
void __init init_IRQ(void)
|
|
|
|
|
{
|
2017-07-31 20:17:03 +00:00
|
|
|
init_irq_stacks();
|
2020-11-30 23:34:42 +00:00
|
|
|
init_irq_scs();
|
2013-01-14 12:39:31 +00:00
|
|
|
irqchip_init();
|
2019-06-11 09:38:12 +00:00
|
|
|
|
|
|
|
|
if (system_uses_irq_prio_masking()) {
|
|
|
|
|
/*
|
|
|
|
|
* Now that we have a stack for our IRQ handler, set
|
|
|
|
|
* the PMR/PSR pair to a consistent state.
|
|
|
|
|
*/
|
|
|
|
|
WARN_ON(read_sysreg(daif) & PSR_A_BIT);
|
|
|
|
|
local_daif_restore(DAIF_PROCCTX_NOIRQ);
|
|
|
|
|
}
|
2012-03-05 11:49:29 +00:00
|
|
|
}
|
