b8889c9c89
We never set this to false. This probably doesn't affect most people's
runtime because GCC will automatically initialize it to false at certain
common optimization levels. But that behavior is related to a bug in
GCC and obviously should not be relied on.
Fixes: 5d6742b377 ("rcu/nocb: Use rcu_segcblist for no-CBs CPUs")
Signed-off-by: Dan Carpenter <dan.carpenter@oracle.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2575 lines
81 KiB
C
2575 lines
81 KiB
C
/* SPDX-License-Identifier: GPL-2.0+ */
|
|
/*
|
|
* Read-Copy Update mechanism for mutual exclusion (tree-based version)
|
|
* Internal non-public definitions that provide either classic
|
|
* or preemptible semantics.
|
|
*
|
|
* Copyright Red Hat, 2009
|
|
* Copyright IBM Corporation, 2009
|
|
*
|
|
* Author: Ingo Molnar <mingo@elte.hu>
|
|
* Paul E. McKenney <paulmck@linux.ibm.com>
|
|
*/
|
|
|
|
#include "../locking/rtmutex_common.h"
|
|
|
|
#ifdef CONFIG_RCU_NOCB_CPU
|
|
static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
|
|
static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
|
|
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
|
|
|
|
/*
|
|
* Check the RCU kernel configuration parameters and print informative
|
|
* messages about anything out of the ordinary.
|
|
*/
|
|
static void __init rcu_bootup_announce_oddness(void)
|
|
{
|
|
if (IS_ENABLED(CONFIG_RCU_TRACE))
|
|
pr_info("\tRCU event tracing is enabled.\n");
|
|
if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
|
|
(!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
|
|
pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d.\n",
|
|
RCU_FANOUT);
|
|
if (rcu_fanout_exact)
|
|
pr_info("\tHierarchical RCU autobalancing is disabled.\n");
|
|
if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
|
|
pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
|
|
if (IS_ENABLED(CONFIG_PROVE_RCU))
|
|
pr_info("\tRCU lockdep checking is enabled.\n");
|
|
if (RCU_NUM_LVLS >= 4)
|
|
pr_info("\tFour(or more)-level hierarchy is enabled.\n");
|
|
if (RCU_FANOUT_LEAF != 16)
|
|
pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
|
|
RCU_FANOUT_LEAF);
|
|
if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
|
|
pr_info("\tBoot-time adjustment of leaf fanout to %d.\n",
|
|
rcu_fanout_leaf);
|
|
if (nr_cpu_ids != NR_CPUS)
|
|
pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%u.\n", NR_CPUS, nr_cpu_ids);
|
|
#ifdef CONFIG_RCU_BOOST
|
|
pr_info("\tRCU priority boosting: priority %d delay %d ms.\n",
|
|
kthread_prio, CONFIG_RCU_BOOST_DELAY);
|
|
#endif
|
|
if (blimit != DEFAULT_RCU_BLIMIT)
|
|
pr_info("\tBoot-time adjustment of callback invocation limit to %ld.\n", blimit);
|
|
if (qhimark != DEFAULT_RCU_QHIMARK)
|
|
pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
|
|
if (qlowmark != DEFAULT_RCU_QLOMARK)
|
|
pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
|
|
if (jiffies_till_first_fqs != ULONG_MAX)
|
|
pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
|
|
if (jiffies_till_next_fqs != ULONG_MAX)
|
|
pr_info("\tBoot-time adjustment of subsequent FQS scan delay to %ld jiffies.\n", jiffies_till_next_fqs);
|
|
if (jiffies_till_sched_qs != ULONG_MAX)
|
|
pr_info("\tBoot-time adjustment of scheduler-enlistment delay to %ld jiffies.\n", jiffies_till_sched_qs);
|
|
if (rcu_kick_kthreads)
|
|
pr_info("\tKick kthreads if too-long grace period.\n");
|
|
if (IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD))
|
|
pr_info("\tRCU callback double-/use-after-free debug enabled.\n");
|
|
if (gp_preinit_delay)
|
|
pr_info("\tRCU debug GP pre-init slowdown %d jiffies.\n", gp_preinit_delay);
|
|
if (gp_init_delay)
|
|
pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_init_delay);
|
|
if (gp_cleanup_delay)
|
|
pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_cleanup_delay);
|
|
if (!use_softirq)
|
|
pr_info("\tRCU_SOFTIRQ processing moved to rcuc kthreads.\n");
|
|
if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG))
|
|
pr_info("\tRCU debug extended QS entry/exit.\n");
|
|
rcupdate_announce_bootup_oddness();
|
|
}
|
|
|
|
#ifdef CONFIG_PREEMPT_RCU
|
|
|
|
static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
|
|
static void rcu_read_unlock_special(struct task_struct *t);
|
|
|
|
/*
|
|
* Tell them what RCU they are running.
|
|
*/
|
|
static void __init rcu_bootup_announce(void)
|
|
{
|
|
pr_info("Preemptible hierarchical RCU implementation.\n");
|
|
rcu_bootup_announce_oddness();
|
|
}
|
|
|
|
/* Flags for rcu_preempt_ctxt_queue() decision table. */
|
|
#define RCU_GP_TASKS 0x8
|
|
#define RCU_EXP_TASKS 0x4
|
|
#define RCU_GP_BLKD 0x2
|
|
#define RCU_EXP_BLKD 0x1
|
|
|
|
/*
|
|
* Queues a task preempted within an RCU-preempt read-side critical
|
|
* section into the appropriate location within the ->blkd_tasks list,
|
|
* depending on the states of any ongoing normal and expedited grace
|
|
* periods. The ->gp_tasks pointer indicates which element the normal
|
|
* grace period is waiting on (NULL if none), and the ->exp_tasks pointer
|
|
* indicates which element the expedited grace period is waiting on (again,
|
|
* NULL if none). If a grace period is waiting on a given element in the
|
|
* ->blkd_tasks list, it also waits on all subsequent elements. Thus,
|
|
* adding a task to the tail of the list blocks any grace period that is
|
|
* already waiting on one of the elements. In contrast, adding a task
|
|
* to the head of the list won't block any grace period that is already
|
|
* waiting on one of the elements.
|
|
*
|
|
* This queuing is imprecise, and can sometimes make an ongoing grace
|
|
* period wait for a task that is not strictly speaking blocking it.
|
|
* Given the choice, we needlessly block a normal grace period rather than
|
|
* blocking an expedited grace period.
|
|
*
|
|
* Note that an endless sequence of expedited grace periods still cannot
|
|
* indefinitely postpone a normal grace period. Eventually, all of the
|
|
* fixed number of preempted tasks blocking the normal grace period that are
|
|
* not also blocking the expedited grace period will resume and complete
|
|
* their RCU read-side critical sections. At that point, the ->gp_tasks
|
|
* pointer will equal the ->exp_tasks pointer, at which point the end of
|
|
* the corresponding expedited grace period will also be the end of the
|
|
* normal grace period.
|
|
*/
|
|
static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
|
|
__releases(rnp->lock) /* But leaves rrupts disabled. */
|
|
{
|
|
int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
|
|
(rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
|
|
(rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
|
|
(rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
|
|
struct task_struct *t = current;
|
|
|
|
raw_lockdep_assert_held_rcu_node(rnp);
|
|
WARN_ON_ONCE(rdp->mynode != rnp);
|
|
WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
|
|
/* RCU better not be waiting on newly onlined CPUs! */
|
|
WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask &
|
|
rdp->grpmask);
|
|
|
|
/*
|
|
* Decide where to queue the newly blocked task. In theory,
|
|
* this could be an if-statement. In practice, when I tried
|
|
* that, it was quite messy.
|
|
*/
|
|
switch (blkd_state) {
|
|
case 0:
|
|
case RCU_EXP_TASKS:
|
|
case RCU_EXP_TASKS + RCU_GP_BLKD:
|
|
case RCU_GP_TASKS:
|
|
case RCU_GP_TASKS + RCU_EXP_TASKS:
|
|
|
|
/*
|
|
* Blocking neither GP, or first task blocking the normal
|
|
* GP but not blocking the already-waiting expedited GP.
|
|
* Queue at the head of the list to avoid unnecessarily
|
|
* blocking the already-waiting GPs.
|
|
*/
|
|
list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
|
|
break;
|
|
|
|
case RCU_EXP_BLKD:
|
|
case RCU_GP_BLKD:
|
|
case RCU_GP_BLKD + RCU_EXP_BLKD:
|
|
case RCU_GP_TASKS + RCU_EXP_BLKD:
|
|
case RCU_GP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
|
|
case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
|
|
|
|
/*
|
|
* First task arriving that blocks either GP, or first task
|
|
* arriving that blocks the expedited GP (with the normal
|
|
* GP already waiting), or a task arriving that blocks
|
|
* both GPs with both GPs already waiting. Queue at the
|
|
* tail of the list to avoid any GP waiting on any of the
|
|
* already queued tasks that are not blocking it.
|
|
*/
|
|
list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
|
|
break;
|
|
|
|
case RCU_EXP_TASKS + RCU_EXP_BLKD:
|
|
case RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
|
|
case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_EXP_BLKD:
|
|
|
|
/*
|
|
* Second or subsequent task blocking the expedited GP.
|
|
* The task either does not block the normal GP, or is the
|
|
* first task blocking the normal GP. Queue just after
|
|
* the first task blocking the expedited GP.
|
|
*/
|
|
list_add(&t->rcu_node_entry, rnp->exp_tasks);
|
|
break;
|
|
|
|
case RCU_GP_TASKS + RCU_GP_BLKD:
|
|
case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
|
|
|
|
/*
|
|
* Second or subsequent task blocking the normal GP.
|
|
* The task does not block the expedited GP. Queue just
|
|
* after the first task blocking the normal GP.
|
|
*/
|
|
list_add(&t->rcu_node_entry, rnp->gp_tasks);
|
|
break;
|
|
|
|
default:
|
|
|
|
/* Yet another exercise in excessive paranoia. */
|
|
WARN_ON_ONCE(1);
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* We have now queued the task. If it was the first one to
|
|
* block either grace period, update the ->gp_tasks and/or
|
|
* ->exp_tasks pointers, respectively, to reference the newly
|
|
* blocked tasks.
|
|
*/
|
|
if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) {
|
|
rnp->gp_tasks = &t->rcu_node_entry;
|
|
WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq);
|
|
}
|
|
if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
|
|
rnp->exp_tasks = &t->rcu_node_entry;
|
|
WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) !=
|
|
!(rnp->qsmask & rdp->grpmask));
|
|
WARN_ON_ONCE(!(blkd_state & RCU_EXP_BLKD) !=
|
|
!(rnp->expmask & rdp->grpmask));
|
|
raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */
|
|
|
|
/*
|
|
* Report the quiescent state for the expedited GP. This expedited
|
|
* GP should not be able to end until we report, so there should be
|
|
* no need to check for a subsequent expedited GP. (Though we are
|
|
* still in a quiescent state in any case.)
|
|
*/
|
|
if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs)
|
|
rcu_report_exp_rdp(rdp);
|
|
else
|
|
WARN_ON_ONCE(rdp->exp_deferred_qs);
|
|
}
|
|
|
|
/*
|
|
* Record a preemptible-RCU quiescent state for the specified CPU.
|
|
* Note that this does not necessarily mean that the task currently running
|
|
* on the CPU is in a quiescent state: Instead, it means that the current
|
|
* grace period need not wait on any RCU read-side critical section that
|
|
* starts later on this CPU. It also means that if the current task is
|
|
* in an RCU read-side critical section, it has already added itself to
|
|
* some leaf rcu_node structure's ->blkd_tasks list. In addition to the
|
|
* current task, there might be any number of other tasks blocked while
|
|
* in an RCU read-side critical section.
|
|
*
|
|
* Callers to this function must disable preemption.
|
|
*/
|
|
static void rcu_qs(void)
|
|
{
|
|
RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n");
|
|
if (__this_cpu_read(rcu_data.cpu_no_qs.s)) {
|
|
trace_rcu_grace_period(TPS("rcu_preempt"),
|
|
__this_cpu_read(rcu_data.gp_seq),
|
|