mirror of
https://github.com/torvalds/linux.git
synced 2024-12-14 23:25:54 +00:00
Merge LKMM and RCU commits
This commit is contained in:
commit
07f038a408
Documentation
MAINTAINERSarch
drivers
include
kernel
locking
rcu
Kconfig.debugrcu.hrcu_segcblist.crcu_segcblist.hrcuperf.crcutorture.csrcutree.ctree.ctree.htree_exp.htree_plugin.htree_stall.hupdate.c
sched
torture.ctrace
net/ipv4
tools/testing/selftests/rcutorture
@ -2129,6 +2129,8 @@ Some of the relevant points of interest are as follows:
|
||||
<li> <a href="#Hotplug CPU">Hotplug CPU</a>.
|
||||
<li> <a href="#Scheduler and RCU">Scheduler and RCU</a>.
|
||||
<li> <a href="#Tracing and RCU">Tracing and RCU</a>.
|
||||
<li> <a href="#Accesses to User Memory and RCU">
|
||||
Accesses to User Memory and RCU</a>.
|
||||
<li> <a href="#Energy Efficiency">Energy Efficiency</a>.
|
||||
<li> <a href="#Scheduling-Clock Interrupts and RCU">
|
||||
Scheduling-Clock Interrupts and RCU</a>.
|
||||
@ -2512,7 +2514,7 @@ disabled across the entire RCU read-side critical section.
|
||||
<p>
|
||||
It is possible to use tracing on RCU code, but tracing itself
|
||||
uses RCU.
|
||||
For this reason, <tt>rcu_dereference_raw_notrace()</tt>
|
||||
For this reason, <tt>rcu_dereference_raw_check()</tt>
|
||||
is provided for use by tracing, which avoids the destructive
|
||||
recursion that could otherwise ensue.
|
||||
This API is also used by virtualization in some architectures,
|
||||
@ -2521,6 +2523,75 @@ cannot be used.
|
||||
The tracing folks both located the requirement and provided the
|
||||
needed fix, so this surprise requirement was relatively painless.
|
||||
|
||||
<h3><a name="Accesses to User Memory and RCU">
|
||||
Accesses to User Memory and RCU</a></h3>
|
||||
|
||||
<p>
|
||||
The kernel needs to access user-space memory, for example, to access
|
||||
data referenced by system-call parameters.
|
||||
The <tt>get_user()</tt> macro does this job.
|
||||
|
||||
<p>
|
||||
However, user-space memory might well be paged out, which means
|
||||
that <tt>get_user()</tt> might well page-fault and thus block while
|
||||
waiting for the resulting I/O to complete.
|
||||
It would be a very bad thing for the compiler to reorder
|
||||
a <tt>get_user()</tt> invocation into an RCU read-side critical
|
||||
section.
|
||||
For example, suppose that the source code looked like this:
|
||||
|
||||
<blockquote>
|
||||
<pre>
|
||||
1 rcu_read_lock();
|
||||
2 p = rcu_dereference(gp);
|
||||
3 v = p->value;
|
||||
4 rcu_read_unlock();
|
||||
5 get_user(user_v, user_p);
|
||||
6 do_something_with(v, user_v);
|
||||
</pre>
|
||||
</blockquote>
|
||||
|
||||
<p>
|
||||
The compiler must not be permitted to transform this source code into
|
||||
the following:
|
||||
|
||||
<blockquote>
|
||||
<pre>
|
||||
1 rcu_read_lock();
|
||||
2 p = rcu_dereference(gp);
|
||||
3 get_user(user_v, user_p); // BUG: POSSIBLE PAGE FAULT!!!
|
||||
4 v = p->value;
|
||||
5 rcu_read_unlock();
|
||||
6 do_something_with(v, user_v);
|
||||
</pre>
|
||||
</blockquote>
|
||||
|
||||
<p>
|
||||
If the compiler did make this transformation in a
|
||||
<tt>CONFIG_PREEMPT=n</tt> kernel build, and if <tt>get_user()</tt> did
|
||||
page fault, the result would be a quiescent state in the middle
|
||||
of an RCU read-side critical section.
|
||||
This misplaced quiescent state could result in line 4 being
|
||||
a use-after-free access, which could be bad for your kernel's
|
||||
actuarial statistics.
|
||||
Similar examples can be constructed with the call to <tt>get_user()</tt>
|
||||
preceding the <tt>rcu_read_lock()</tt>.
|
||||
|
||||
<p>
|
||||
Unfortunately, <tt>get_user()</tt> doesn't have any particular
|
||||
ordering properties, and in some architectures the underlying <tt>asm</tt>
|
||||
isn't even marked <tt>volatile</tt>.
|
||||
And even if it was marked <tt>volatile</tt>, the above access to
|
||||
<tt>p->value</tt> is not volatile, so the compiler would not have any
|
||||
reason to keep those two accesses in order.
|
||||
|
||||
<p>
|
||||
Therefore, the Linux-kernel definitions of <tt>rcu_read_lock()</tt>
|
||||
and <tt>rcu_read_unlock()</tt> must act as compiler barriers,
|
||||
at least for outermost instances of <tt>rcu_read_lock()</tt> and
|
||||
<tt>rcu_read_unlock()</tt> within a nested set of RCU read-side critical
|
||||
sections.
|
||||
|
||||
<h3><a name="Energy Efficiency">Energy Efficiency</a></h3>
|
||||
|
||||
<p>
|
||||
|
@ -57,6 +57,12 @@ o A CPU-bound real-time task in a CONFIG_PREEMPT_RT kernel that
|
||||
CONFIG_PREEMPT_RCU case, you might see stall-warning
|
||||
messages.
|
||||
|
||||
You can use the rcutree.kthread_prio kernel boot parameter to
|
||||
increase the scheduling priority of RCU's kthreads, which can
|
||||
help avoid this problem. However, please note that doing this
|
||||
can increase your system's context-switch rate and thus degrade
|
||||
performance.
|
||||
|
||||
o A periodic interrupt whose handler takes longer than the time
|
||||
interval between successive pairs of interrupts. This can
|
||||
prevent RCU's kthreads and softirq handlers from running.
|
||||
|
@ -3837,12 +3837,13 @@
|
||||
RCU_BOOST is not set, valid values are 0-99 and
|
||||
the default is zero (non-realtime operation).
|
||||
|
||||
rcutree.rcu_nocb_leader_stride= [KNL]
|
||||
Set the number of NOCB kthread groups, which
|
||||
defaults to the square root of the number of
|
||||
CPUs. Larger numbers reduces the wakeup overhead
|
||||
on the per-CPU grace-period kthreads, but increases
|
||||
that same overhead on each group's leader.
|
||||
rcutree.rcu_nocb_gp_stride= [KNL]
|
||||
Set the number of NOCB callback kthreads in
|
||||
each group, which defaults to the square root
|
||||
of the number of CPUs. Larger numbers reduce
|
||||
the wakeup overhead on the global grace-period
|
||||
kthread, but increases that same overhead on
|
||||
each group's NOCB grace-period kthread.
|
||||
|
||||
rcutree.qhimark= [KNL]
|
||||
Set threshold of queued RCU callbacks beyond which
|
||||
@ -4047,6 +4048,10 @@
|
||||
rcutorture.verbose= [KNL]
|
||||
Enable additional printk() statements.
|
||||
|
||||
rcupdate.rcu_cpu_stall_ftrace_dump= [KNL]
|
||||
Dump ftrace buffer after reporting RCU CPU
|
||||
stall warning.
|
||||
|
||||
rcupdate.rcu_cpu_stall_suppress= [KNL]
|
||||
Suppress RCU CPU stall warning messages.
|
||||
|
||||
|
@ -9326,7 +9326,7 @@ F: drivers/misc/lkdtm/*
|
||||
|
||||
LINUX KERNEL MEMORY CONSISTENCY MODEL (LKMM)
|
||||
M: Alan Stern <stern@rowland.harvard.edu>
|
||||
M: Andrea Parri <andrea.parri@amarulasolutions.com>
|
||||
M: Andrea Parri <parri.andrea@gmail.com>
|
||||
M: Will Deacon <will@kernel.org>
|
||||
M: Peter Zijlstra <peterz@infradead.org>
|
||||
M: Boqun Feng <boqun.feng@gmail.com>
|
||||
|
@ -264,15 +264,13 @@ int __cpu_disable(void)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static DECLARE_COMPLETION(cpu_died);
|
||||
|
||||
/*
|
||||
* called on the thread which is asking for a CPU to be shutdown -
|
||||
* waits until shutdown has completed, or it is timed out.
|
||||
*/
|
||||
void __cpu_die(unsigned int cpu)
|
||||
{
|
||||
if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) {
|
||||
if (!cpu_wait_death(cpu, 5)) {
|
||||
pr_err("CPU%u: cpu didn't die\n", cpu);
|
||||
return;
|
||||
}
|
||||
@ -319,7 +317,7 @@ void arch_cpu_idle_dead(void)
|
||||
* this returns, power and/or clocks can be removed at any point
|
||||
* from this CPU and its cache by platform_cpu_kill().
|
||||
*/
|
||||
complete(&cpu_died);
|
||||
(void)cpu_report_death();
|
||||
|
||||
/*
|
||||
* Ensure that the cache lines associated with that completion are
|
||||
|
@ -535,7 +535,7 @@ static inline void note_hpte_modification(struct kvm *kvm,
|
||||
*/
|
||||
static inline struct kvm_memslots *kvm_memslots_raw(struct kvm *kvm)
|
||||
{
|
||||
return rcu_dereference_raw_notrace(kvm->memslots[0]);
|
||||
return rcu_dereference_raw_check(kvm->memslots[0]);
|
||||
}
|
||||
|
||||
extern void kvmppc_mmu_debugfs_init(struct kvm *kvm);
|
||||
|
@ -29,6 +29,7 @@
|
||||
static bool pci_mmcfg_running_state;
|
||||
static bool pci_mmcfg_arch_init_failed;
|
||||
static DEFINE_MUTEX(pci_mmcfg_lock);
|
||||
#define pci_mmcfg_lock_held() lock_is_held(&(pci_mmcfg_lock).dep_map)
|
||||
|
||||
LIST_HEAD(pci_mmcfg_list);
|
||||
|
||||
@ -54,7 +55,7 @@ static void list_add_sorted(struct pci_mmcfg_region *new)
|
||||
struct pci_mmcfg_region *cfg;
|
||||
|
||||
/* keep list sorted by segment and starting bus number */
|
||||
list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list) {
|
||||
list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list, pci_mmcfg_lock_held()) {
|
||||
if (cfg->segment > new->segment ||
|
||||
(cfg->segment == new->segment &&
|
||||
cfg->start_bus >= new->start_bus)) {
|
||||
@ -118,7 +119,7 @@ struct pci_mmcfg_region *pci_mmconfig_lookup(int segment, int bus)
|
||||
{
|
||||
struct pci_mmcfg_region *cfg;
|
||||
|
||||
list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list)
|
||||
list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list, pci_mmcfg_lock_held())
|
||||
if (cfg->segment == segment &&
|
||||
cfg->start_bus <= bus && bus <= cfg->end_bus)
|
||||
return cfg;
|
||||
|
@ -14,6 +14,7 @@
|
||||
#include <linux/slab.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/lockdep.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/kmod.h>
|
||||
@ -80,6 +81,7 @@ struct acpi_ioremap {
|
||||
|
||||
static LIST_HEAD(acpi_ioremaps);
|
||||
static DEFINE_MUTEX(acpi_ioremap_lock);
|
||||
#define acpi_ioremap_lock_held() lock_is_held(&acpi_ioremap_lock.dep_map)
|
||||
|
||||
static void __init acpi_request_region (struct acpi_generic_address *gas,
|
||||
unsigned int length, char *desc)
|
||||
@ -206,7 +208,7 @@ acpi_map_lookup(acpi_physical_address phys, acpi_size size)
|
||||
{
|
||||
struct acpi_ioremap *map;
|
||||
|
||||
list_for_each_entry_rcu(map, &acpi_ioremaps, list)
|
||||
list_for_each_entry_rcu(map, &acpi_ioremaps, list, acpi_ioremap_lock_held())
|
||||
if (map->phys <= phys &&
|
||||
phys + size <= map->phys + map->size)
|
||||
return map;
|
||||
@ -249,7 +251,7 @@ acpi_map_lookup_virt(void __iomem *virt, acpi_size size)
|
||||
{
|
||||
struct acpi_ioremap *map;
|
||||
|
||||
list_for_each_entry_rcu(map, &acpi_ioremaps, list)
|
||||
list_for_each_entry_rcu(map, &acpi_ioremaps, list, acpi_ioremap_lock_held())
|
||||
if (map->virt <= virt &&
|
||||
virt + size <= map->virt + map->size)
|
||||
return map;
|
||||
|
@ -165,6 +165,7 @@ static inline int devtmpfs_init(void) { return 0; }
|
||||
/* Device links support */
|
||||
extern int device_links_read_lock(void);
|
||||
extern void device_links_read_unlock(int idx);
|
||||
extern int device_links_read_lock_held(void);
|
||||
extern int device_links_check_suppliers(struct device *dev);
|
||||
extern void device_links_driver_bound(struct device *dev);
|
||||
extern void device_links_driver_cleanup(struct device *dev);
|
||||
|
@ -68,6 +68,11 @@ void device_links_read_unlock(int idx)
|
||||
{
|
||||
srcu_read_unlock(&device_links_srcu, idx);
|
||||
}
|
||||
|
||||
int device_links_read_lock_held(void)
|
||||
{
|
||||
return srcu_read_lock_held(&device_links_srcu);
|
||||
}
|
||||
#else /* !CONFIG_SRCU */
|
||||
static DECLARE_RWSEM(device_links_lock);
|
||||
|
||||
@ -91,6 +96,13 @@ void device_links_read_unlock(int not_used)
|
||||
{
|
||||
up_read(&device_links_lock);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
||||
int device_links_read_lock_held(void)
|
||||
{
|
||||
return lockdep_is_held(&device_links_lock);
|
||||
}
|
||||
#endif
|
||||
#endif /* !CONFIG_SRCU */
|
||||
|
||||
/**
|
||||
|
@ -287,7 +287,8 @@ static int rpm_get_suppliers(struct device *dev)
|
||||
{
|
||||
struct device_link *link;
|
||||
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
|
||||
device_links_read_lock_held()) {
|
||||
int retval;
|
||||
|
||||
if (!(link->flags & DL_FLAG_PM_RUNTIME) ||
|
||||
@ -309,7 +310,8 @@ static void rpm_put_suppliers(struct device *dev)
|
||||
{
|
||||
struct device_link *link;
|
||||
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
|
||||
device_links_read_lock_held()) {
|
||||
if (READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND)
|
||||
continue;
|
||||
|
||||
@ -1640,7 +1642,8 @@ void pm_runtime_clean_up_links(struct device *dev)
|
||||
|
||||
idx = device_links_read_lock();
|
||||
|
||||
list_for_each_entry_rcu(link, &dev->links.consumers, s_node) {
|
||||
list_for_each_entry_rcu(link, &dev->links.consumers, s_node,
|
||||
device_links_read_lock_held()) {
|
||||
if (link->flags & DL_FLAG_STATELESS)
|
||||
continue;
|
||||
|
||||
@ -1662,7 +1665,8 @@ void pm_runtime_get_suppliers(struct device *dev)
|
||||
|
||||
idx = device_links_read_lock();
|
||||
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
|
||||
device_links_read_lock_held())
|
||||
if (link->flags & DL_FLAG_PM_RUNTIME) {
|
||||
link->supplier_preactivated = true;
|
||||
refcount_inc(&link->rpm_active);
|
||||
@ -1683,7 +1687,8 @@ void pm_runtime_put_suppliers(struct device *dev)
|
||||
|
||||
idx = device_links_read_lock();
|
||||
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
|
||||
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
|
||||
device_links_read_lock_held())
|
||||
if (link->supplier_preactivated) {
|
||||
link->supplier_preactivated = false;
|
||||
if (refcount_dec_not_one(&link->rpm_active))
|
||||
|
@ -14,6 +14,9 @@
|
||||
#ifndef __INCLUDE_LINUX_RCU_SEGCBLIST_H
|
||||
#define __INCLUDE_LINUX_RCU_SEGCBLIST_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/atomic.h>
|
||||
|
||||
/* Simple unsegmented callback lists. */
|
||||
struct rcu_cblist {
|
||||
struct rcu_head *head;
|
||||
@ -65,8 +68,14 @@ struct rcu_segcblist {
|
||||
struct rcu_head *head;
|
||||
struct rcu_head **tails[RCU_CBLIST_NSEGS];
|
||||
unsigned long gp_seq[RCU_CBLIST_NSEGS];
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
atomic_long_t len;
|
||||
#else
|
||||
long len;
|
||||
#endif
|
||||
long len_lazy;
|
||||
u8 enabled;
|
||||
u8 offloaded;
|
||||
};
|
||||
|
||||
#define RCU_SEGCBLIST_INITIALIZER(n) \
|
||||
|
@ -31,9 +31,7 @@ struct rcu_sync {
|
||||
*/
|
||||
static inline bool rcu_sync_is_idle(struct rcu_sync *rsp)
|
||||
{
|
||||
RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&
|
||||
!rcu_read_lock_bh_held() &&
|
||||
!rcu_read_lock_sched_held(),
|
||||
RCU_LOCKDEP_WARN(!rcu_read_lock_any_held(),
|
||||
"suspicious rcu_sync_is_idle() usage");
|
||||
return !READ_ONCE(rsp->gp_state); /* GP_IDLE */
|
||||
}
|
||||
|
@ -40,6 +40,24 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
|
||||
*/
|
||||
#define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
|
||||
|
||||
/*
|
||||
* Check during list traversal that we are within an RCU reader
|
||||
*/
|
||||
|
||||
#define check_arg_count_one(dummy)
|
||||
|
||||
#ifdef CONFIG_PROVE_RCU_LIST
|
||||
#define __list_check_rcu(dummy, cond, extra...) \
|
||||
({ \
|
||||
check_arg_count_one(extra); \
|
||||
RCU_LOCKDEP_WARN(!cond && !rcu_read_lock_any_held(), \
|
||||
"RCU-list traversed in non-reader section!"); \
|
||||
})
|
||||
#else
|
||||
#define __list_check_rcu(dummy, cond, extra...) \
|
||||
({ check_arg_count_one(extra); })
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Insert a new entry between two known consecutive entries.
|
||||
*
|
||||
@ -343,14 +361,16 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
|
||||
* @pos: the type * to use as a loop cursor.
|
||||
* @head: the head for your list.
|
||||
* @member: the name of the list_head within the struct.
|
||||
* @cond: optional lockdep expression if called from non-RCU protection.
|
||||
*
|
||||
* This list-traversal primitive may safely run concurrently with
|
||||
* the _rcu list-mutation primitives such as list_add_rcu()
|
||||
* as long as the traversal is guarded by rcu_read_lock().
|
||||
*/
|
||||
#define list_for_each_entry_rcu(pos, head, member) \
|
||||
for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
|
||||
&pos->member != (head); \
|
||||
#define list_for_each_entry_rcu(pos, head, member, cond...) \
|
||||
for (__list_check_rcu(dummy, ## cond, 0), \
|
||||
pos = list_entry_rcu((head)->next, typeof(*pos), member); \
|
||||
&pos->member != (head); \
|
||||
pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
|
||||
|
||||
/**
|
||||
@ -616,13 +636,15 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
|
||||
* @pos: the type * to use as a loop cursor.
|
||||
* @head: the head for your list.
|
||||
* @member: the name of the hlist_node within the struct.
|
||||
* @cond: optional lockdep expression if called from non-RCU protection.
|
||||
*
|
||||
* This list-traversal primitive may safely run concurrently with
|
||||
* the _rcu list-mutation primitives such as hlist_add_head_rcu()
|
||||
* as long as the traversal is guarded by rcu_read_lock().
|
||||
*/
|
||||
#define hlist_for_each_entry_rcu(pos, head, member) \
|
||||
for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\
|
||||
#define hlist_for_each_entry_rcu(pos, head, member, cond...) \
|
||||
for (__list_check_rcu(dummy, ## cond, 0), \
|
||||
pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
|
||||
typeof(*(pos)), member); \
|
||||
pos; \
|
||||
pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
|
||||
@ -642,10 +664,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
|
||||
* not do any RCU debugging or tracing.
|
||||
*/
|
||||
#define hlist_for_each_entry_rcu_notrace(pos, head, member) \
|
||||
for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\
|
||||
for (pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_first_rcu(head)),\
|
||||
typeof(*(pos)), member); \
|
||||
pos; \
|
||||
pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\
|
||||
pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_next_rcu(\
|
||||
&(pos)->member)), typeof(*(pos)), member))
|
||||
|
||||
/**
|
||||
|
@ -221,6 +221,7 @@ int debug_lockdep_rcu_enabled(void);
|
||||
int rcu_read_lock_held(void);
|
||||
int rcu_read_lock_bh_held(void);
|
||||
int rcu_read_lock_sched_held(void);
|
||||
int rcu_read_lock_any_held(void);
|
||||
|
||||
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
||||
|
||||
@ -241,6 +242,12 @@ static inline int rcu_read_lock_sched_held(void)
|
||||
{
|
||||
return !preemptible();
|
||||
}
|
||||
|
||||
static inline int rcu_read_lock_any_held(void)
|
||||
{
|
||||
return !preemptible();
|
||||
}
|
||||
|
||||
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
||||
|
||||
#ifdef CONFIG_PROVE_RCU
|
||||
@ -476,7 +483,7 @@ do { \
|
||||
* The no-tracing version of rcu_dereference_raw() must not call
|
||||
* rcu_read_lock_held().
|
||||
*/
|
||||
#define rcu_dereference_raw_notrace(p) __rcu_dereference_check((p), 1, __rcu)
|
||||
#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu)
|
||||
|
||||
/**
|
||||
* rcu_dereference_protected() - fetch RCU pointer when updates prevented
|
||||
|
@ -100,7 +100,6 @@ TRACE_EVENT_RCU(rcu_grace_period,
|
||||
* "Startedroot": Requested a nocb grace period based on root-node data.
|
||||
* "NoGPkthread": The RCU grace-period kthread has not yet started.
|
||||
* "StartWait": Start waiting for the requested grace period.
|
||||
* "ResumeWait": Resume waiting after signal.
|
||||
* "EndWait": Complete wait.
|
||||
* "Cleanup": Clean up rcu_node structure after previous GP.
|
||||
* "CleanupMore": Clean up, and another GP is needed.
|
||||
@ -267,7 +266,8 @@ TRACE_EVENT_RCU(rcu_exp_funnel_lock,
|
||||
* "WakeNotPoll": Don't wake rcuo kthread because it is polling.
|
||||
* "DeferredWake": Carried out the "IsDeferred" wakeup.
|
||||
* "Poll": Start of new polling cycle for rcu_nocb_poll.
|
||||
* "Sleep": Sleep waiting for CBs for !rcu_nocb_poll.
|
||||
* "Sleep": Sleep waiting for GP for !rcu_nocb_poll.
|
||||
* "CBSleep": Sleep waiting for CBs for !rcu_nocb_poll.
|
||||
* "WokeEmpty": rcuo kthread woke to find empty list.
|
||||
* "WokeNonEmpty": rcuo kthread woke to find non-empty list.
|
||||
* "WaitQueue": Enqueue partially done, timed wait for it to complete.
|
||||
|
@ -620,7 +620,7 @@ static void print_lock(struct held_lock *hlock)
|
||||
return;
|
||||
}
|
||||
|
||||
printk(KERN_CONT "%p", hlock->instance);
|
||||
printk(KERN_CONT "%px", hlock->instance);
|
||||
print_lock_name(lock);
|
||||
printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);
|
||||
}
|
||||
|
@ -8,6 +8,17 @@ menu "RCU Debugging"
|
||||
config PROVE_RCU
|
||||
def_bool PROVE_LOCKING
|
||||
|
||||
config PROVE_RCU_LIST
|
||||
bool "RCU list lockdep debugging"
|
||||
depends on PROVE_RCU && RCU_EXPERT
|
||||
default n
|
||||
help
|
||||
Enable RCU lockdep checking for list usages. By default it is
|
||||
turned off since there are several list RCU users that still
|
||||
need to be converted to pass a lockdep expression. To prevent
|
||||
false-positive splats, we keep it default disabled but once all
|
||||
users are converted, we can remove this config option.
|
||||
|
||||
config TORTURE_TEST
|
||||
tristate
|
||||
default n
|
||||
|
@ -227,6 +227,7 @@ static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)
|
||||
|
||||
#ifdef CONFIG_RCU_STALL_COMMON
|
||||
|
||||
extern int rcu_cpu_stall_ftrace_dump;
|
||||
extern int rcu_cpu_stall_suppress;
|
||||
extern int rcu_cpu_stall_timeout;
|
||||
int rcu_jiffies_till_stall_check(void);
|
||||
|
@ -23,6 +23,49 @@ void rcu_cblist_init(struct rcu_cblist *rclp)
|
||||
rclp->len_lazy = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Enqueue an rcu_head structure onto the specified callback list.
|
||||
* This function assumes that the callback is non-lazy because it
|
||||
* is intended for use by no-CBs CPUs, which do not distinguish
|
||||
* between lazy and non-lazy RCU callbacks.
|
||||
*/
|
||||
void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp)
|
||||
{
|
||||
*rclp->tail = rhp;
|
||||
rclp->tail = &rhp->next;
|
||||
WRITE_ONCE(rclp->len, rclp->len + 1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush the second rcu_cblist structure onto the first one, obliterating
|
||||
* any contents of the first. If rhp is non-NULL, enqueue it as the sole
|
||||
* element of the second rcu_cblist structure, but ensuring that the second
|
||||
* rcu_cblist structure, if initially non-empty, always appears non-empty
|
||||
* throughout the process. If rdp is NULL, the second rcu_cblist structure
|
||||
* is instead initialized to empty.
|
||||
*/
|
||||
void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
|
||||
struct rcu_cblist *srclp,
|
||||
struct rcu_head *rhp)
|
||||
{
|
||||
drclp->head = srclp->head;
|
||||
if (drclp->head)
|
||||
drclp->tail = srclp->tail;
|
||||
else
|
||||
drclp->tail = &drclp->head;
|
||||
drclp->len = srclp->len;
|
||||
drclp->len_lazy = srclp->len_lazy;
|
||||
if (!rhp) {
|
||||
rcu_cblist_init(srclp);
|
||||
} else {
|
||||
rhp->next = NULL;
|
||||
srclp->head = rhp;
|
||||
srclp->tail = &rhp->next;
|
||||
WRITE_ONCE(srclp->len, 1);
|
||||
srclp->len_lazy = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Dequeue the oldest rcu_head structure from the specified callback
|
||||
* list. This function assumes that the callback is non-lazy, but
|
||||
@ -44,6 +87,67 @@ struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)
|
||||
return rhp;
|
||||
}
|
||||
|
||||
/* Set the length of an rcu_segcblist structure. */
|
||||
void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)
|
||||
{
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
atomic_long_set(&rsclp->len, v);
|
||||
#else
|
||||
WRITE_ONCE(rsclp->len, v);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Increase the numeric length of an rcu_segcblist structure by the
|
||||
* specified amount, which can be negative. This can cause the ->len
|
||||
* field to disagree with the actual number of callbacks on the structure.
|
||||
* This increase is fully ordered with respect to the callers accesses
|
||||
* both before and after.
|
||||
*/
|
||||
void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
|
||||
{
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
smp_mb__before_atomic(); /* Up to the caller! */
|
||||
atomic_long_add(v, &rsclp->len);
|
||||
smp_mb__after_atomic(); /* Up to the caller! */
|
||||
#else
|
||||
smp_mb(); /* Up to the caller! */
|
||||
WRITE_ONCE(rsclp->len, rsclp->len + v);
|
||||
smp_mb(); /* Up to the caller! */
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Increase the numeric length of an rcu_segcblist structure by one.
|
||||
* This can cause the ->len field to disagree with the actual number of
|
||||
* callbacks on the structure. This increase is fully ordered with respect
|
||||
* to the callers accesses both before and after.
|
||||
*/
|
||||
void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
rcu_segcblist_add_len(rsclp, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Exchange the numeric length of the specified rcu_segcblist structure
|
||||
* with the specified value. This can cause the ->len field to disagree
|
||||
* with the actual number of callbacks on the structure. This exchange is
|
||||
* fully ordered with respect to the callers accesses both before and after.
|
||||
*/
|
||||
long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v)
|
||||
{
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
return atomic_long_xchg(&rsclp->len, v);
|
||||
#else
|
||||
long ret = rsclp->len;
|
||||
|
||||
smp_mb(); /* Up to the caller! */
|
||||
WRITE_ONCE(rsclp->len, v);
|
||||
smp_mb(); /* Up to the caller! */
|
||||
return ret;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Initialize an rcu_segcblist structure.
|
||||
*/
|
||||
@ -56,8 +160,9 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp)
|
||||
rsclp->head = NULL;
|
||||
for (i = 0; i < RCU_CBLIST_NSEGS; i++)
|
||||
rsclp->tails[i] = &rsclp->head;
|
||||
rsclp->len = 0;
|
||||
rcu_segcblist_set_len(rsclp, 0);
|
||||
rsclp->len_lazy = 0;
|
||||
rsclp->enabled = 1;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -69,7 +174,16 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp)
|
||||
WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));
|
||||
WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));
|
||||
WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp));
|
||||
rsclp->tails[RCU_NEXT_TAIL] = NULL;
|
||||
rsclp->enabled = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Mark the specified rcu_segcblist structure as offloaded. This
|
||||
* structure must be empty.
|
||||
*/
|
||||
void rcu_segcblist_offload(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
rsclp->offloaded = 1;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -117,6 +231,18 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return false if there are no CBs awaiting grace periods, otherwise,
|
||||
* return true and store the nearest waited-upon grace period into *lp.
|
||||
*/
|
||||
bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)
|
||||
{
|
||||
if (!rcu_segcblist_pend_cbs(rsclp))
|
||||
return false;
|
||||
*lp = rsclp->gp_seq[RCU_WAIT_TAIL];
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Enqueue the specified callback onto the specified rcu_segcblist
|
||||
* structure, updating accounting as needed. Note that the ->len
|
||||
@ -129,13 +255,13 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)
|
||||
void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
|
||||
struct rcu_head *rhp, bool lazy)
|
||||
{
|
||||
WRITE_ONCE(rsclp->len, rsclp->len + 1); /* ->len sampled locklessly. */
|
||||
rcu_segcblist_inc_len(rsclp);
|
||||
if (lazy)
|
||||
rsclp->len_lazy++;
|
||||
smp_mb(); /* Ensure counts are updated before callback is enqueued. */
|
||||
rhp->next = NULL;
|
||||
*rsclp->tails[RCU_NEXT_TAIL] = rhp;
|
||||
rsclp->tails[RCU_NEXT_TAIL] = &rhp->next;
|
||||
WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
|
||||
WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -155,7 +281,7 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
|
||||
|
||||
if (rcu_segcblist_n_cbs(rsclp) == 0)
|
||||
return false;
|
||||
WRITE_ONCE(rsclp->len, rsclp->len + 1);
|
||||
rcu_segcblist_inc_len(rsclp);
|
||||
if (lazy)
|
||||
rsclp->len_lazy++;
|
||||
smp_mb(); /* Ensure counts are updated before callback is entrained. */
|
||||
@ -163,9 +289,9 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
|
||||
for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
|
||||
if (rsclp->tails[i] != rsclp->tails[i - 1])
|
||||
break;
|
||||
*rsclp->tails[i] = rhp;
|
||||
WRITE_ONCE(*rsclp->tails[i], rhp);
|
||||
for (; i <= RCU_NEXT_TAIL; i++)
|
||||
rsclp->tails[i] = &rhp->next;
|
||||
WRITE_ONCE(rsclp->tails[i], &rhp->next);
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -182,9 +308,8 @@ void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
|
||||
struct rcu_cblist *rclp)
|
||||
{
|
||||
rclp->len_lazy += rsclp->len_lazy;
|
||||
rclp->len += rsclp->len;
|
||||
rsclp->len_lazy = 0;
|
||||
WRITE_ONCE(rsclp->len, 0); /* ->len sampled locklessly. */
|
||||
rclp->len = rcu_segcblist_xchg_len(rsclp, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -200,12 +325,12 @@ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
|
||||
if (!rcu_segcblist_ready_cbs(rsclp))
|
||||
return; /* Nothing to do. */
|
||||
*rclp->tail = rsclp->head;
|
||||
rsclp->head = *rsclp->tails[RCU_DONE_TAIL];
|
||||
*rsclp->tails[RCU_DONE_TAIL] = NULL;
|
||||
WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]);
|
||||
WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
|
||||
rclp->tail = rsclp->tails[RCU_DONE_TAIL];
|
||||
for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)
|
||||
if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
|
||||
rsclp->tails[i] = &rsclp->head;
|
||||
WRITE_ONCE(rsclp->tails[i], &rsclp->head);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -224,9 +349,9 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
|
||||
return; /* Nothing to do. */
|
||||
*rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
|
||||
rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
|
||||
*rsclp->tails[RCU_DONE_TAIL] = NULL;
|
||||
WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
|
||||
for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)
|
||||
rsclp->tails[i] = rsclp->tails[RCU_DONE_TAIL];
|
||||
WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -237,8 +362,7 @@ void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
|
||||
struct rcu_cblist *rclp)
|
||||
{
|
||||
rsclp->len_lazy += rclp->len_lazy;
|
||||
/* ->len sampled locklessly. */
|
||||
WRITE_ONCE(rsclp->len, rsclp->len + rclp->len);
|
||||
rcu_segcblist_add_len(rsclp, rclp->len);
|
||||
rclp->len_lazy = 0;
|
||||
rclp->len = 0;
|
||||
}
|
||||
@ -255,10 +379,10 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
|
||||
if (!rclp->head)
|
||||
return; /* No callbacks to move. */
|
||||
*rclp->tail = rsclp->head;
|
||||
rsclp->head = rclp->head;
|
||||
WRITE_ONCE(rsclp->head, rclp->head);
|
||||
for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)
|
||||
if (&rsclp->head == rsclp->tails[i])
|
||||
rsclp->tails[i] = rclp->tail;
|
||||
WRITE_ONCE(rsclp->tails[i], rclp->tail);
|
||||
else
|
||||
break;
|
||||
rclp->head = NULL;
|
||||
@ -274,8 +398,8 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
|
||||
{
|
||||
if (!rclp->head)
|
||||
return; /* Nothing to do. */
|
||||
*rsclp->tails[RCU_NEXT_TAIL] = rclp->head;
|
||||
rsclp->tails[RCU_NEXT_TAIL] = rclp->tail;
|
||||
WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
|
||||
WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
|
||||
rclp->head = NULL;
|
||||
rclp->tail = &rclp->head;
|
||||
}
|
||||
@ -299,7 +423,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
|
||||
for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
|
||||
if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
|
||||
break;
|
||||
rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i];
|
||||
WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);
|
||||
}
|
||||
|
||||
/* If no callbacks moved, nothing more need be done. */
|
||||
@ -308,7 +432,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
|
||||
|
||||
/* Clean up tail pointers that might have been misordered above. */
|
||||
for (j = RCU_WAIT_TAIL; j < i; j++)
|
||||
rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL];
|
||||
WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);
|
||||
|
||||
/*
|
||||
* Callbacks moved, so clean up the misordered ->tails[] pointers
|
||||
@ -319,7 +443,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
|
||||
for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
|
||||
if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
|
||||
break; /* No more callbacks. */
|
||||
rsclp->tails[j] = rsclp->tails[i];
|
||||
WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);
|
||||
rsclp->gp_seq[j] = rsclp->gp_seq[i];
|
||||
}
|
||||
}
|
||||
@ -384,7 +508,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
|
||||
* structure other than in the RCU_NEXT_TAIL segment.
|
||||
*/
|
||||
for (; i < RCU_NEXT_TAIL; i++) {
|
||||
rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL];
|
||||
WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);
|
||||
rsclp->gp_seq[i] = seq;
|
||||
}
|
||||
return true;
|
||||
|
@ -9,6 +9,12 @@
|
||||
|
||||
#include <linux/rcu_segcblist.h>
|
||||
|
||||
/* Return number of callbacks in the specified callback list. */
|
||||
static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
|
||||
{
|
||||
return READ_ONCE(rclp->len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Account for the fact that a previously dequeued callback turned out
|
||||
* to be marked as lazy.
|
||||
@ -19,6 +25,10 @@ static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp)
|
||||
}
|
||||
|
||||
void rcu_cblist_init(struct rcu_cblist *rclp);
|
||||
void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
|
||||
void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
|
||||
struct rcu_cblist *srclp,
|
||||
struct rcu_head *rhp);
|
||||
struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
|
||||
|
||||
/*
|
||||
@ -36,13 +46,17 @@ struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
|
||||
*/
|
||||
static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
return !rsclp->head;
|
||||
return !READ_ONCE(rsclp->head);
|
||||
}
|
||||
|
||||
/* Return number of callbacks in segmented callback list. */
|
||||
static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
return atomic_long_read(&rsclp->len);
|
||||
#else
|
||||
return READ_ONCE(rsclp->len);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Return number of lazy callbacks in segmented callback list. */
|
||||
@ -54,16 +68,22 @@ static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
|
||||
/* Return number of lazy callbacks in segmented callback list. */
|
||||
static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
return rsclp->len - rsclp->len_lazy;
|
||||
return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy;
|
||||
}
|
||||
|
||||
/*
|
||||
* Is the specified rcu_segcblist enabled, for example, not corresponding
|
||||
* to an offline or callback-offloaded CPU?
|
||||
* to an offline CPU?
|
||||
*/
|
||||
static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
return !!rsclp->tails[RCU_NEXT_TAIL];
|
||||
return rsclp->enabled;
|
||||
}
|
||||
|
||||
/* Is the specified rcu_segcblist offloaded? */
|
||||
static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
return rsclp->offloaded;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -73,36 +93,18 @@ static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
|
||||
*/
|
||||
static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
|
||||
{
|
||||
return !*rsclp->tails[seg];
|
||||
}
|
||||
|
||||
/*
|
||||
* Interim function to return rcu_segcblist head pointer. Longer term, the
|
||||
* rcu_segcblist will be used more pervasively, removing the need for this
|
||||
* function.
|
||||
*/
|
||||
static inline struct rcu_head *rcu_segcblist_head(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
return rsclp->head;
|
||||
}
|
||||
|
||||
/*
|
||||
* Interim function to return rcu_segcblist head pointer. Longer term, the
|
||||
* rcu_segcblist will be used more pervasively, removing the need for this
|
||||
* function.
|
||||
*/
|
||||
static inline struct rcu_head **rcu_segcblist_tail(struct rcu_segcblist *rsclp)
|
||||
{
|
||||
WARN_ON_ONCE(rcu_segcblist_empty(rsclp));
|
||||
return rsclp->tails[RCU_NEXT_TAIL];
|
||||
return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
|
||||
}
|
||||
|
||||
void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
|
||||
void rcu_segcblist_init(struct rcu_segcblist *rsclp);
|
||||
void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
|
||||
void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
|
||||
bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
|
||||
bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
|
||||
struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
|
||||
struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
|
||||
bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
|
||||
void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
|
||||
struct rcu_head *rhp, bool lazy);
|
||||
bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
|
||||
|
@ -89,7 +89,7 @@ torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable
|
||||
|
||||
static char *perf_type = "rcu";
|
||||
module_param(perf_type, charp, 0444);
|
||||
MODULE_PARM_DESC(perf_type, "Type of RCU to performance-test (rcu, rcu_bh, ...)");
|
||||
MODULE_PARM_DESC(perf_type, "Type of RCU to performance-test (rcu, srcu, ...)");
|
||||
|
||||
static int nrealreaders;
|
||||
static int nrealwriters;
|
||||
@ -375,6 +375,14 @@ rcu_perf_writer(void *arg)
|
||||
if (holdoff)
|
||||
schedule_timeout_uninterruptible(holdoff * HZ);
|
||||
|
||||
/*
|
||||
* Wait until rcu_end_inkernel_boot() is called for normal GP tests
|
||||
* so that RCU is not always expedited for normal GP tests.
|
||||
* The system_state test is approximate, but works well in practice.
|
||||
*/
|
||||
while (!gp_exp && system_state != SYSTEM_RUNNING)
|
||||
schedule_timeout_uninterruptible(1);
|
||||
|
||||
t = ktime_get_mono_fast_ns();
|
||||
if (atomic_inc_return(&n_rcu_perf_writer_started) >= nrealwriters) {
|
||||
t_rcu_perf_writer_started = t;
|
||||
|
@ -161,6 +161,7 @@ static atomic_long_t n_rcu_torture_timers;
|
||||
static long n_barrier_attempts;
|
||||
static long n_barrier_successes; /* did rcu_barrier test succeed? */
|
||||
static struct list_head rcu_torture_removed;
|
||||
static unsigned long shutdown_jiffies;
|
||||
|
||||
static int rcu_torture_writer_state;
|
||||
#define RTWS_FIXED_DELAY 0
|
||||
@ -228,6 +229,15 @@ static u64 notrace rcu_trace_clock_local(void)
|
||||
}
|
||||
#endif /* #else #ifdef CONFIG_RCU_TRACE */
|
||||
|
||||
/*
|
||||
* Stop aggressive CPU-hog tests a bit before the end of the test in order
|
||||
* to avoid interfering with test shutdown.
|
||||
*/
|
||||
static bool shutdown_time_arrived(void)
|
||||
{
|
||||
return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ);
|
||||
}
|
||||
|
||||
static unsigned long boost_starttime; /* jiffies of next boost test start. */
|
||||
static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */
|
||||
/* and boost task create/destroy. */
|
||||
@ -1713,12 +1723,14 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
|
||||
}
|
||||
|
||||
// Give the scheduler a chance, even on nohz_full CPUs.
|
||||
static void rcu_torture_fwd_prog_cond_resched(void)
|
||||
static void rcu_torture_fwd_prog_cond_resched(unsigned long iter)
|
||||
{
|
||||
if (IS_ENABLED(CONFIG_PREEMPT) && IS_ENABLED(CONFIG_NO_HZ_FULL)) {
|
||||
if (need_resched())
|
||||
// Real call_rcu() floods hit userspace, so emulate that.
|
||||
if (need_resched() || (iter & 0xfff))
|
||||
schedule();
|
||||
} else {
|
||||
// No userspace emulation: CB invocation throttles call_rcu()
|
||||
cond_resched();
|
||||
}
|
||||
}
|
||||
@ -1746,7 +1758,7 @@ static unsigned long rcu_torture_fwd_prog_cbfree(void)
|
||||
spin_unlock_irqrestore(&rcu_fwd_lock, flags);
|
||||
kfree(rfcp);
|
||||
freed++;
|
||||
rcu_torture_fwd_prog_cond_resched();
|
||||
rcu_torture_fwd_prog_cond_resched(freed);
|
||||
}
|
||||
return freed;
|
||||
}
|
||||
@ -1785,15 +1797,17 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries)
|
||||
WRITE_ONCE(rcu_fwd_startat, jiffies);
|
||||
stopat = rcu_fwd_startat + dur;
|
||||
while (time_before(jiffies, stopat) &&
|
||||
!shutdown_time_arrived() &&
|
||||
!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
|
||||
idx = cur_ops->readlock();
|
||||
udelay(10);
|
||||
cur_ops->readunlock(idx);
|
||||
if (!fwd_progress_need_resched || need_resched())
|
||||
rcu_torture_fwd_prog_cond_resched();
|
||||
rcu_torture_fwd_prog_cond_resched(1);
|
||||
}
|
||||
(*tested_tries)++;
|
||||
if (!time_before(jiffies, stopat) &&
|
||||
!shutdown_time_arrived() &&
|
||||
!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
|
||||
(*tested)++;
|
||||
cver = READ_ONCE(rcu_torture_current_version) - cver;
|
||||
@ -1852,6 +1866,7 @@ static void rcu_torture_fwd_prog_cr(void)
|
||||
gps = cur_ops->get_gp_seq();
|
||||
rcu_launder_gp_seq_start = gps;
|
||||
while (time_before(jiffies, stopat) &&
|
||||
!shutdown_time_arrived() &&
|
||||
!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
|
||||
rfcp = READ_ONCE(rcu_fwd_cb_head);
|
||||
rfcpn = NULL;
|
||||
@ -1875,7 +1890,7 @@ static void rcu_torture_fwd_prog_cr(void)
|
||||
rfcp->rfc_gps = 0;
|
||||
}
|
||||
cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
|
||||
rcu_torture_fwd_prog_cond_resched();
|
||||
rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
|
||||
}
|
||||
stoppedat = jiffies;
|
||||
n_launders_cb_snap = READ_ONCE(n_launders_cb);
|
||||
@ -1884,7 +1899,8 @@ static void rcu_torture_fwd_prog_cr(void)
|
||||
cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
|
||||
(void)rcu_torture_fwd_prog_cbfree();
|
||||
|
||||
if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop)) {
|
||||
if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) &&
|
||||
!shutdown_time_arrived()) {
|
||||
WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED);
|
||||
pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n",
|
||||
__func__,
|
||||
@ -2160,6 +2176,7 @@ rcu_torture_cleanup(void)
|
||||
return;
|
||||
}
|
||||
|
||||
show_rcu_gp_kthreads();
|
||||
rcu_torture_barrier_cleanup();
|
||||
torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_task);
|
||||
torture_stop_kthread(rcu_torture_stall, stall_task);
|
||||
@ -2465,6 +2482,7 @@ rcu_torture_init(void)
|
||||
goto unwind;
|
||||
rcutor_hp = firsterr;
|
||||
}
|
||||
shutdown_jiffies = jiffies + shutdown_secs * HZ;
|
||||
firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
|
||||
if (firsterr)
|
||||
goto unwind;
|
||||
|
@ -1279,8 +1279,9 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)
|
||||
|
||||
c0 = l0 - u0;
|
||||
c1 = l1 - u1;
|
||||
pr_cont(" %d(%ld,%ld %1p)",
|
||||
cpu, c0, c1, rcu_segcblist_head(&sdp->srcu_cblist));
|
||||
pr_cont(" %d(%ld,%ld %c)",
|
||||
cpu, c0, c1,
|
||||
"C."[rcu_segcblist_empty(&sdp->srcu_cblist)]);
|
||||
s0 += c0;
|
||||
s1 += c1;
|
||||
}
|
||||
|
@ -56,6 +56,7 @@
|
||||
#include <linux/smpboot.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/sched/isolation.h>
|
||||
#include <linux/sched/clock.h>
|
||||
#include "../time/tick-internal.h"
|
||||
|
||||
#include "tree.h"
|
||||
@ -210,9 +211,9 @@ static long rcu_get_n_cbs_cpu(int cpu)
|
||||
{
|
||||
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
|
||||
|
||||
if (rcu_segcblist_is_enabled(&rdp->cblist)) /* Online normal CPU? */
|
||||
if (rcu_segcblist_is_enabled(&rdp->cblist))
|
||||
return rcu_segcblist_n_cbs(&rdp->cblist);
|
||||
return rcu_get_n_cbs_nocb_cpu(rdp); /* Works for offline, too. */
|
||||
return 0;
|
||||
}
|
||||
|
||||
void rcu_softirq_qs(void)
|
||||
@ -416,6 +417,12 @@ module_param(qlowmark, long, 0444);
|
||||
static ulong jiffies_till_first_fqs = ULONG_MAX;
|
||||
static ulong jiffies_till_next_fqs = ULONG_MAX;
|
||||
static bool rcu_kick_kthreads;
|
||||
static int rcu_divisor = 7;
|
||||
module_param(rcu_divisor, int, 0644);
|
||||
|
||||
/* Force an exit from rcu_do_batch() after 3 milliseconds. */
|
||||
static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
|
||||
module_param(rcu_resched_ns, long, 0644);
|
||||
|
||||
/*
|
||||
* How long the grace period must be before we start recruiting
|
||||
@ -1251,6 +1258,7 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
|
||||
unsigned long gp_seq_req;
|
||||
bool ret = false;
|
||||
|
||||
rcu_lockdep_assert_cblist_protected(rdp);
|
||||
raw_lockdep_assert_held_rcu_node(rnp);
|
||||
|
||||
/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
|
||||
@ -1292,7 +1300,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
|
||||
unsigned long c;
|
||||
bool needwake;
|
||||
|
||||
lockdep_assert_irqs_disabled();
|
||||
rcu_lockdep_assert_cblist_protected(rdp);
|
||||
c = rcu_seq_snap(&rcu_state.gp_seq);
|
||||
if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
|
||||
/* Old request still live, so mark recent callbacks. */
|
||||
@ -1318,6 +1326,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
|
||||
*/
|
||||
static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
|
||||
{
|
||||
rcu_lockdep_assert_cblist_protected(rdp);
|
||||
raw_lockdep_assert_held_rcu_node(rnp);
|
||||
|
||||
/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
|
||||
@ -1334,6 +1343,21 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
|
||||
return rcu_accelerate_cbs(rnp, rdp);
|
||||
}
|
||||
|
||||
/*
|
||||
* Move and classify callbacks, but only if doing so won't require
|
||||
* that the RCU grace-period kthread be awakened.
|
||||
*/
|
||||
static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
|
||||
struct rcu_data *rdp)
|
||||
{
|
||||
rcu_lockdep_assert_cblist_protected(rdp);
|
||||
if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) ||
|
||||
!raw_spin_trylock_rcu_node(rnp))
|
||||
return;
|
||||
WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
|
||||
raw_spin_unlock_rcu_node(rnp);
|
||||
}
|
||||
|
||||
/*
|
||||
* Update CPU-local rcu_data state to record the beginnings and ends of
|
||||
* grace periods. The caller must hold the ->lock of the leaf rcu_node
|
||||
@ -1342,8 +1366,10 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
|
||||
*/
|
||||
static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
|
||||
{
|
||||
bool ret;
|
||||
bool ret = false;
|
||||
bool need_gp;
|
||||
const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
rcu_segcblist_is_offloaded(&rdp->cblist);
|
||||
|
||||
raw_lockdep_assert_held_rcu_node(rnp);
|
||||
|
||||
@ -1353,10 +1379,12 @@ static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
|
||||
/* Handle the ends of any preceding grace periods first. */
|
||||
if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||
|
||||
unlikely(READ_ONCE(rdp->gpwrap))) {
|
||||
ret = rcu_advance_cbs(rnp, rdp); /* Advance callbacks. */
|
||||
if (!offloaded)
|
||||
ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
|
||||
trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
|
||||
} else {
|
||||
ret = rcu_accelerate_cbs(rnp, rdp); /* Recent callbacks. */
|
||||
if (!offloaded)
|
||||
ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
|
||||
}
|
||||
|
||||
/* Now handle the beginnings of any new-to-this-CPU grace periods. */
|
||||
@ -1657,6 +1685,7 @@ static void rcu_gp_cleanup(void)
|
||||
unsigned long gp_duration;
|
||||
bool needgp = false;
|
||||
unsigned long new_gp_seq;
|
||||
bool offloaded;
|
||||
struct rcu_data *rdp;
|
||||
struct rcu_node *rnp = rcu_get_root();
|
||||
struct swait_queue_head *sq;
|
||||
@ -1722,7 +1751,9 @@ static void rcu_gp_cleanup(void)
|
||||
needgp = true;
|
||||
}
|
||||
/* Advance CBs to reduce false positives below. */
|
||||
if (!rcu_accelerate_cbs(rnp, rdp) && needgp) {
|
||||
offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
rcu_segcblist_is_offloaded(&rdp->cblist);
|
||||
if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
|
||||
WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
|
||||
rcu_state.gp_req_activity = jiffies;
|
||||
trace_rcu_grace_period(rcu_state.name,
|
||||
@ -1916,7 +1947,9 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp)
|
||||
{
|
||||
unsigned long flags;
|
||||
unsigned long mask;
|
||||
bool needwake;
|
||||
bool needwake = false;
|
||||
const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
rcu_segcblist_is_offloaded(&rdp->cblist);
|
||||
struct rcu_node *rnp;
|
||||
|
||||
rnp = rdp->mynode;
|
||||
@ -1943,7 +1976,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp)
|
||||
* This GP can't end until cpu checks in, so all of our
|
||||
* callbacks can be processed during the next GP.
|
||||
*/
|
||||
needwake = rcu_accelerate_cbs(rnp, rdp);
|
||||
if (!offloaded)
|
||||
needwake = rcu_accelerate_cbs(rnp, rdp);
|
||||
|
||||
rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
|
||||
/* ^^^ Released rnp->lock */
|
||||
@ -2077,9 +2111,12 @@ int rcutree_dead_cpu(unsigned int cpu)
|
||||
static void rcu_do_batch(struct rcu_data *rdp)
|
||||
{
|
||||
unsigned long flags;
|
||||
const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
rcu_segcblist_is_offloaded(&rdp->cblist);
|
||||
struct rcu_head *rhp;
|
||||
struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
|
||||
long bl, count;
|
||||
long pending, tlimit = 0;
|
||||
|
||||
/* If no callbacks are ready, just return. */
|
||||
if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
|
||||
@ -2099,13 +2136,19 @@ static void rcu_do_batch(struct rcu_data *rdp)
|
||||
* callback counts, as rcu_barrier() needs to be conservative.
|
||||
*/
|
||||
local_irq_save(flags);
|
||||
rcu_nocb_lock(rdp);
|
||||
WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
|
||||
bl = rdp->blimit;
|
||||
pending = rcu_segcblist_n_cbs(&rdp->cblist);
|
||||
bl = max(rdp->blimit, pending >> rcu_divisor);
|
||||
if (unlikely(bl > 100))
|
||||
tlimit = local_clock() + rcu_resched_ns;
|
||||
trace_rcu_batch_start(rcu_state.name,
|
||||
rcu_segcblist_n_lazy_cbs(&rdp->cblist),
|
||||
rcu_segcblist_n_cbs(&rdp->cblist), bl);
|
||||
rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
|
||||
local_irq_restore(flags);
|
||||
if (offloaded)
|
||||
rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
|
||||
rcu_nocb_unlock_irqrestore(rdp, flags);
|
||||
|
||||
/* Invoke callbacks. */
|
||||
rhp = rcu_cblist_dequeue(&rcl);
|
||||
@ -2117,13 +2160,29 @@ static void rcu_do_batch(struct rcu_data *rdp)
|
||||
* Stop only if limit reached and CPU has something to do.
|
||||
* Note: The rcl structure counts down from zero.
|
||||
*/
|
||||
if (-rcl.len >= bl &&
|
||||
if (-rcl.len >= bl && !offloaded &&
|
||||
(need_resched() ||
|
||||
(!is_idle_task(current) && !rcu_is_callbacks_kthread())))
|
||||
break;
|
||||
if (unlikely(tlimit)) {
|
||||
/* only call local_clock() every 32 callbacks */
|
||||
if (likely((-rcl.len & 31) || local_clock() < tlimit))
|
||||
continue;
|
||||
/* Exceeded the time limit, so leave. */
|
||||
break;
|
||||
}
|
||||
if (offloaded) {
|
||||
WARN_ON_ONCE(in_serving_softirq());
|
||||
local_bh_enable();
|
||||
lockdep_assert_irqs_enabled();
|
||||
cond_resched_tasks_rcu_qs();
|
||||
lockdep_assert_irqs_enabled();
|
||||
local_bh_disable();
|
||||
}
|
||||
}
|
||||
|
||||
local_irq_save(flags);
|
||||
rcu_nocb_lock(rdp);
|
||||
count = -rcl.len;
|
||||
trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
|
||||
is_idle_task(current), rcu_is_callbacks_kthread());
|
||||
@ -2149,12 +2208,14 @@ static void rcu_do_batch(struct rcu_data *rdp)
|
||||
* The following usually indicates a double call_rcu(). To track
|
||||
* this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
|
||||
*/
|
||||
WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0));
|
||||
WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
|
||||
WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
count != 0 && rcu_segcblist_empty(&rdp->cblist));
|
||||
|
||||
local_irq_restore(flags);
|
||||
rcu_nocb_unlock_irqrestore(rdp, flags);
|
||||
|
||||
/* Re-invoke RCU core processing if there are callbacks remaining. */
|
||||
if (rcu_segcblist_ready_cbs(&rdp->cblist))
|
||||
if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
|
||||
invoke_rcu_core();
|
||||
}
|
||||
|
||||
@ -2280,6 +2341,8 @@ static __latent_entropy void rcu_core(void)
|
||||
unsigned long flags;
|
||||
struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
|
||||
struct rcu_node *rnp = rdp->mynode;
|
||||
const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
rcu_segcblist_is_offloaded(&rdp->cblist);
|
||||
|
||||
if (cpu_is_offline(smp_processor_id()))
|
||||
return;
|
||||
@ -2299,7 +2362,7 @@ static __latent_entropy void rcu_core(void)
|
||||
|
||||
/* No grace period and unregistered callbacks? */
|
||||
if (!rcu_gp_in_progress() &&
|
||||
rcu_segcblist_is_enabled(&rdp->cblist)) {
|
||||
rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
|
||||
local_irq_save(flags);
|
||||
if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
|
||||
rcu_accelerate_cbs_unlocked(rnp, rdp);
|
||||
@ -2309,7 +2372,7 @@ static __latent_entropy void rcu_core(void)
|
||||
rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
|
||||
|
||||
/* If there are callbacks ready, invoke them. */
|
||||
if (rcu_segcblist_ready_cbs(&rdp->cblist) &&
|
||||
if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
|
||||
likely(READ_ONCE(rcu_scheduler_fully_active)))
|
||||
rcu_do_batch(rdp);
|
||||
|
||||
@ -2489,10 +2552,11 @@ static void rcu_leak_callback(struct rcu_head *rhp)
|
||||
* is expected to specify a CPU.
|
||||
*/
|
||||
static void
|
||||
__call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy)
|
||||
__call_rcu(struct rcu_head *head, rcu_callback_t func, bool lazy)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct rcu_data *rdp;
|
||||
bool was_alldone;
|
||||
|
||||
/* Misaligned rcu_head! */
|
||||
WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
|
||||
@ -2514,28 +2578,18 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy)
|
||||
rdp = this_cpu_ptr(&rcu_data);
|
||||
|
||||
/* Add the callback to our list. */
|
||||
if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) || cpu != -1) {
|
||||
int offline;
|
||||
|
||||
if (cpu != -1)
|
||||
rdp = per_cpu_ptr(&rcu_data, cpu);
|
||||
if (likely(rdp->mynode)) {
|
||||
/* Post-boot, so this should be for a no-CBs CPU. */
|
||||
offline = !__call_rcu_nocb(rdp, head, lazy, flags);
|
||||
WARN_ON_ONCE(offline);
|
||||
/* Offline CPU, _call_rcu() illegal, leak callback. */
|
||||
local_irq_restore(flags);
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* Very early boot, before rcu_init(). Initialize if needed
|
||||
* and then drop through to queue the callback.
|
||||
*/
|
||||
WARN_ON_ONCE(cpu != -1);
|
||||
if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
|
||||
// This can trigger due to call_rcu() from offline CPU:
|
||||
WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
|
||||
WARN_ON_ONCE(!rcu_is_watching());
|
||||
// Very early boot, before rcu_init(). Initialize if needed
|
||||
// and then drop through to queue the callback.
|
||||
if (rcu_segcblist_empty(&rdp->cblist))
|
||||
rcu_segcblist_init(&rdp->cblist);
|
||||
}
|
||||
if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
|
||||
return; // Enqueued onto ->nocb_bypass, so just leave.
|
||||
/* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */
|
||||
rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
|
||||
if (__is_kfree_rcu_offset((unsigned long)func))
|
||||
trace_rcu_kfree_callback(rcu_state.name, head,
|
||||
@ -2548,8 +2602,13 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy)
|
||||
rcu_segcblist_n_cbs(&rdp->cblist));
|
||||
|
||||
/* Go handle any RCU core processing required. */
|
||||
__call_rcu_core(rdp, head, flags);
|
||||
local_irq_restore(flags);
|
||||
if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
|
||||
unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
|
||||
__call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
|
||||
} else {
|
||||
__call_rcu_core(rdp, head, flags);
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2589,7 +2648,7 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy)
|
||||
*/
|
||||
void call_rcu(struct rcu_head *head, rcu_callback_t func)
|
||||
{
|
||||
__call_rcu(head, func, -1, 0);
|
||||
__call_rcu(head, func, 0);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(call_rcu);
|
||||
|
||||
@ -2602,7 +2661,7 @@ EXPORT_SYMBOL_GPL(call_rcu);
|
||||
*/
|
||||
void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
|
||||
{
|
||||
__call_rcu(head, func, -1, 1);
|
||||
__call_rcu(head, func, 1);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kfree_call_rcu);
|
||||
|
||||
@ -2735,6 +2794,10 @@ static int rcu_pending(void)
|
||||
/* Check for CPU stalls, if enabled. */
|
||||
check_cpu_stall(rdp);
|
||||
|
||||
/* Does this CPU need a deferred NOCB wakeup? */
|
||||
if (rcu_nocb_need_deferred_wakeup(rdp))
|
||||
return 1;
|
||||
|
||||
/* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
|
||||
if (rcu_nohz_full_cpu())
|
||||
return 0;
|
||||
@ -2750,6 +2813,8 @@ static int rcu_pending(void)
|
||||
/* Has RCU gone idle with this CPU needing another grace period? */
|
||||
if (!rcu_gp_in_progress() &&
|
||||
rcu_segcblist_is_enabled(&rdp->cblist) &&
|
||||
(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) ||
|
||||
!rcu_segcblist_is_offloaded(&rdp->cblist)) &&
|
||||
!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
|
||||
return 1;
|
||||
|
||||
@ -2758,10 +2823,6 @@ static int rcu_pending(void)
|
||||
unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
|
||||
return 1;
|
||||
|
||||
/* Does this CPU need a deferred NOCB wakeup? */
|
||||
if (rcu_nocb_need_deferred_wakeup(rdp))
|
||||
return 1;
|
||||
|
||||
/* nothing to do */
|
||||
return 0;
|
||||
}
|
||||
@ -2801,6 +2862,8 @@ static void rcu_barrier_func(void *unused)
|
||||
rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
|
||||
rdp->barrier_head.func = rcu_barrier_callback;
|
||||
debug_rcu_head_queue(&rdp->barrier_head);
|
||||
rcu_nocb_lock(rdp);
|
||||
WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
|
||||
if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) {
|
||||
atomic_inc(&rcu_state.barrier_cpu_count);
|
||||
} else {
|
||||
@ -2808,6 +2871,7 @@ static void rcu_barrier_func(void *unused)
|
||||
rcu_barrier_trace(TPS("IRQNQ"), -1,
|
||||
rcu_state.barrier_sequence);
|
||||
}
|
||||
rcu_nocb_unlock(rdp);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2858,22 +2922,11 @@ void rcu_barrier(void)
|
||||
* corresponding CPU's preceding callbacks have been invoked.
|
||||
*/
|
||||
for_each_possible_cpu(cpu) {
|
||||
if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu))
|
||||
continue;
|
||||
rdp = per_cpu_ptr(&rcu_data, cpu);
|
||||
if (rcu_is_nocb_cpu(cpu)) {
|
||||
if (!rcu_nocb_cpu_needs_barrier(cpu)) {
|
||||
rcu_barrier_trace(TPS("OfflineNoCB"), cpu,
|
||||
rcu_state.barrier_sequence);
|
||||
} else {
|
||||
rcu_barrier_trace(TPS("OnlineNoCB"), cpu,
|
||||
rcu_state.barrier_sequence);
|
||||
smp_mb__before_atomic();
|
||||
atomic_inc(&rcu_state.barrier_cpu_count);
|
||||
__call_rcu(&rdp->barrier_head,
|
||||
rcu_barrier_callback, cpu, 0);
|
||||
}
|
||||
} else if (rcu_segcblist_n_cbs(&rdp->cblist)) {
|
||||
if (!cpu_online(cpu) &&
|
||||
!rcu_segcblist_is_offloaded(&rdp->cblist))
|
||||
continue;
|
||||
if (rcu_segcblist_n_cbs(&rdp->cblist)) {
|
||||
rcu_barrier_trace(TPS("OnlineQ"), cpu,
|
||||
rcu_state.barrier_sequence);
|
||||
smp_call_function_single(cpu, rcu_barrier_func, NULL, 1);
|
||||
@ -2958,7 +3011,8 @@ rcu_boot_init_percpu_data(int cpu)
|
||||
* Initializes a CPU's per-CPU RCU data. Note that only one online or
|
||||
* offline event can be happening at a given time. Note also that we can
|
||||
* accept some slop in the rsp->gp_seq access due to the fact that this
|
||||
* CPU cannot possibly have any RCU callbacks in flight yet.
|
||||
* CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
|
||||
* And any offloaded callbacks are being numbered elsewhere.
|
||||
*/
|
||||
int rcutree_prepare_cpu(unsigned int cpu)
|
||||
{
|
||||
@ -2972,7 +3026,7 @@ int rcutree_prepare_cpu(unsigned int cpu)
|
||||
rdp->n_force_qs_snap = rcu_state.n_force_qs;
|
||||
rdp->blimit = blimit;
|
||||
if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
|
||||
!init_nocb_callback_list(rdp))
|
||||
!rcu_segcblist_is_offloaded(&rdp->cblist))
|
||||
rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */
|
||||
rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
|
||||
rcu_dynticks_eqs_online();
|
||||
@ -3151,29 +3205,38 @@ void rcutree_migrate_callbacks(int cpu)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct rcu_data *my_rdp;
|
||||
struct rcu_node *my_rnp;
|
||||
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
|
||||
struct rcu_node *rnp_root = rcu_get_root();
|
||||
bool needwake;
|
||||
|
||||
if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist))
|
||||
if (rcu_segcblist_is_offloaded(&rdp->cblist) ||
|
||||
rcu_segcblist_empty(&rdp->cblist))
|
||||
return; /* No callbacks to migrate. */
|
||||
|
||||
local_irq_save(flags);
|
||||
my_rdp = this_cpu_ptr(&rcu_data);
|
||||
if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
|
||||
local_irq_restore(flags);
|
||||
return;
|
||||
}
|
||||
raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
|
||||
my_rnp = my_rdp->mynode;
|
||||
rcu_nocb_lock(my_rdp); /* irqs already disabled. */
|
||||
WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
|
||||
raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
|
||||
/* Leverage recent GPs and set GP for new callbacks. */
|
||||
needwake = rcu_advance_cbs(rnp_root, rdp) ||
|
||||
rcu_advance_cbs(rnp_root, my_rdp);
|
||||
needwake = rcu_advance_cbs(my_rnp, rdp) ||
|
||||
rcu_advance_cbs(my_rnp, my_rdp);
|
||||
rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
|
||||
needwake = needwake || rcu_advance_cbs(my_rnp, my_rdp);
|
||||
rcu_segcblist_disable(&rdp->cblist);
|
||||
WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
|
||||
!rcu_segcblist_n_cbs(&my_rdp->cblist));
|
||||
raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags);
|
||||
if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
|
||||
raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
|
||||
__call_rcu_nocb_wake(my_rdp, true, flags);
|
||||
} else {
|
||||
rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
|
||||
raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
|
||||
}
|
||||
if (needwake)
|
||||
rcu_gp_kthread_wake();
|
||||
lockdep_assert_irqs_enabled();
|
||||
WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 ||
|
||||
!rcu_segcblist_empty(&rdp->cblist),
|
||||
"rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
|
||||
|
@ -194,29 +194,38 @@ struct rcu_data {
|
||||
|
||||
/* 5) Callback offloading. */
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
struct rcu_head *nocb_head; /* CBs waiting for kthread. */
|
||||
struct rcu_head **nocb_tail;
|
||||
atomic_long_t nocb_q_count; /* # CBs waiting for nocb */
|
||||
atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */
|
||||
struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */
|
||||
struct rcu_head **nocb_follower_tail;
|
||||
struct swait_queue_head nocb_wq; /* For nocb kthreads to sleep on. */
|
||||
struct task_struct *nocb_kthread;
|
||||
struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */
|
||||
struct task_struct *nocb_gp_kthread;
|
||||
raw_spinlock_t nocb_lock; /* Guard following pair of fields. */
|
||||
atomic_t nocb_lock_contended; /* Contention experienced. */
|
||||
int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */
|
||||
struct timer_list nocb_timer; /* Enforce finite deferral. */
|
||||
unsigned long nocb_gp_adv_time; /* Last call_rcu() CB adv (jiffies). */
|
||||
|
||||
/* The following fields are used by the leader, hence own cacheline. */
|
||||
struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp;
|
||||
/* CBs waiting for GP. */
|
||||
struct rcu_head **nocb_gp_tail;
|
||||
bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */
|
||||
struct rcu_data *nocb_next_follower;
|
||||
/* Next follower in wakeup chain. */
|
||||
/* The following fields are used by call_rcu, hence own cacheline. */
|
||||
raw_spinlock_t nocb_bypass_lock ____cacheline_internodealigned_in_smp;
|
||||
struct rcu_cblist nocb_bypass; /* Lock-contention-bypass CB list. */
|
||||
unsigned long nocb_bypass_first; /* Time (jiffies) of first enqueue. */
|
||||
unsigned long nocb_nobypass_last; /* Last ->cblist enqueue (jiffies). */
|
||||
int nocb_nobypass_count; /* # ->cblist enqueues at ^^^ time. */
|
||||
|
||||
/* The following fields are used by the follower, hence new cachline. */
|
||||
struct rcu_data *nocb_leader ____cacheline_internodealigned_in_smp;
|
||||
/* Leader CPU takes GP-end wakeups. */
|
||||
/* The following fields are used by GP kthread, hence own cacheline. */
|
||||
raw_spinlock_t nocb_gp_lock ____cacheline_internodealigned_in_smp;
|
||||
struct timer_list nocb_bypass_timer; /* Force nocb_bypass flush. */
|
||||
u8 nocb_gp_sleep; /* Is the nocb GP thread asleep? */
|
||||
u8 nocb_gp_bypass; /* Found a bypass on last scan? */
|
||||
u8 nocb_gp_gp; /* GP to wait for on last scan? */
|
||||
unsigned long nocb_gp_seq; /* If so, ->gp_seq to wait for. */
|
||||
unsigned long nocb_gp_loops; /* # passes through wait code. */
|
||||
struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */
|
||||
bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */
|
||||
struct task_struct *nocb_cb_kthread;
|
||||
struct rcu_data *nocb_next_cb_rdp;
|
||||
/* Next rcu_data in wakeup chain. */
|
||||
|
||||
/* The following fields are used by CB kthread, hence new cacheline. */
|
||||
struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp;
|
||||
/* GP rdp takes GP-end wakeups. */
|
||||
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
|
||||
|
||||
/* 6) RCU priority boosting. */
|
||||
@ -419,25 +428,39 @@ static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
|
||||
static bool rcu_preempt_need_deferred_qs(struct task_struct *t);
|
||||
static void rcu_preempt_deferred_qs(struct task_struct *t);
|
||||
static void zero_cpu_stall_ticks(struct rcu_data *rdp);
|
||||
static bool rcu_nocb_cpu_needs_barrier(int cpu);
|
||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
|
||||
static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
|
||||
static void rcu_init_one_nocb(struct rcu_node *rnp);
|
||||
static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
|
||||
bool lazy, unsigned long flags);
|
||||
static bool rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,
|
||||
struct rcu_data *rdp,
|
||||
unsigned long flags);
|
||||
static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
|
||||
unsigned long j);
|
||||
static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
|
||||
bool *was_alldone, unsigned long flags);
|
||||
static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
|
||||
unsigned long flags);
|
||||
static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
|
||||
static void do_nocb_deferred_wakeup(struct rcu_data *rdp);
|
||||
static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
|
||||
static void rcu_spawn_cpu_nocb_kthread(int cpu);
|
||||
static void __init rcu_spawn_nocb_kthreads(void);
|
||||
static void show_rcu_nocb_state(struct rcu_data *rdp);
|
||||
static void rcu_nocb_lock(struct rcu_data *rdp);
|
||||
static void rcu_nocb_unlock(struct rcu_data *rdp);
|
||||
static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
|
||||
unsigned long flags);
|
||||
static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
static void __init rcu_organize_nocb_kthreads(void);
|
||||
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
|
||||
static bool init_nocb_callback_list(struct rcu_data *rdp);
|
||||
static unsigned long rcu_get_n_cbs_nocb_cpu(struct rcu_data *rdp);
|
||||
#define rcu_nocb_lock_irqsave(rdp, flags) \
|
||||
do { \
|
||||
if (!rcu_segcblist_is_offloaded(&(rdp)->cblist)) \
|
||||
local_irq_save(flags); \
|
||||
else \
|
||||
raw_spin_lock_irqsave(&(rdp)->nocb_lock, (flags)); \
|
||||
} while (0)
|
||||
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
|
||||
#define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags)
|
||||
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
|
||||
|
||||
static void rcu_bind_gp_kthread(void);
|
||||
static bool rcu_nohz_full_cpu(void);
|
||||
static void rcu_dynticks_task_enter(void);
|
||||
|
@ -781,7 +781,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp)
|
||||
* other hand, if the CPU is not in an RCU read-side critical section,
|
||||
* the IPI handler reports the quiescent state immediately.
|
||||
*
|
||||
* Although this is a greate improvement over previous expedited
|
||||
* Although this is a great improvement over previous expedited
|
||||
* implementations, it is still unfriendly to real-time workloads, so is
|
||||
* thus not recommended for any sort of common-case code. In fact, if
|
||||
* you are using synchronize_rcu_expedited() in a loop, please restructure
|
||||
@ -792,6 +792,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp)
|
||||
*/
|
||||
void synchronize_rcu_expedited(void)
|
||||
{
|
||||
bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT);
|
||||
struct rcu_exp_work rew;
|
||||
struct rcu_node *rnp;
|
||||
unsigned long s;
|
||||
@ -817,7 +818,7 @@ void synchronize_rcu_expedited(void)
|
||||
return; /* Someone else did our work for us. */
|
||||
|
||||
/* Ensure that load happens before action based on it. */
|
||||
if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) {
|
||||
if (unlikely(boottime)) {
|
||||
/* Direct call during scheduler init and early_initcalls(). */
|
||||
rcu_exp_sel_wait_wake(s);
|
||||
} else {
|
||||
@ -835,5 +836,8 @@ void synchronize_rcu_expedited(void)
|
||||
|
||||
/* Let the next expedited grace period start. */
|
||||
mutex_unlock(&rcu_state.exp_mutex);
|
||||
|
||||
if (likely(!boottime))
|
||||
destroy_work_on_stack(&rew.rew_work);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -527,6 +527,8 @@ static void check_cpu_stall(struct rcu_data *rdp)
|
||||
|
||||
/* We haven't checked in, so go dump stack. */
|
||||
print_cpu_stall();
|
||||
if (rcu_cpu_stall_ftrace_dump)
|
||||
rcu_ftrace_dump(DUMP_ALL);
|
||||
|
||||
} else if (rcu_gp_in_progress() &&
|
||||
ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
|
||||
@ -534,6 +536,8 @@ static void check_cpu_stall(struct rcu_data *rdp)
|
||||
|
||||
/* They had a few time units to dump stack, so complain. */
|
||||
print_other_cpu_stall(gs2);
|
||||
if (rcu_cpu_stall_ftrace_dump)
|
||||
rcu_ftrace_dump(DUMP_ALL);
|
||||
}
|
||||
}
|
||||
|
||||
@ -585,6 +589,11 @@ void show_rcu_gp_kthreads(void)
|
||||
cpu, (long)rdp->gp_seq_needed);
|
||||
}
|
||||
}
|
||||
for_each_possible_cpu(cpu) {
|
||||
rdp = per_cpu_ptr(&rcu_data, cpu);
|
||||
if (rcu_segcblist_is_offloaded(&rdp->cblist))
|
||||
show_rcu_nocb_state(rdp);
|
||||
}
|
||||
/* sched_show_task(rcu_state.gp_kthread); */
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
|
||||
|
@ -61,9 +61,15 @@ module_param(rcu_normal_after_boot, int, 0);
|
||||
|
||||
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
||||
/**
|
||||
* rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
|
||||
* rcu_read_lock_held_common() - might we be in RCU-sched read-side critical section?
|
||||
* @ret: Best guess answer if lockdep cannot be relied on
|
||||
*
|
||||
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
|
||||
* Returns true if lockdep must be ignored, in which case *ret contains
|
||||
* the best guess described below. Otherwise returns false, in which
|
||||
* case *ret tells the caller nothing and the caller should instead
|
||||
* consult lockdep.
|
||||
*
|
||||
* If CONFIG_DEBUG_LOCK_ALLOC is selected, set *ret to nonzero iff in an
|
||||
* RCU-sched read-side critical section. In absence of
|
||||
* CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
|
||||
* critical section unless it can prove otherwise. Note that disabling
|
||||
@ -75,35 +81,45 @@ module_param(rcu_normal_after_boot, int, 0);
|
||||
* Check debug_lockdep_rcu_enabled() to prevent false positives during boot
|
||||
* and while lockdep is disabled.
|
||||
*
|
||||
* Note that if the CPU is in the idle loop from an RCU point of
|
||||
* view (ie: that we are in the section between rcu_idle_enter() and
|
||||
* rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU
|
||||
* did an rcu_read_lock(). The reason for this is that RCU ignores CPUs
|
||||
* that are in such a section, considering these as in extended quiescent
|
||||
* state, so such a CPU is effectively never in an RCU read-side critical
|
||||
* section regardless of what RCU primitives it invokes. This state of
|
||||
* affairs is required --- we need to keep an RCU-free window in idle
|
||||
* where the CPU may possibly enter into low power mode. This way we can
|
||||
* notice an extended quiescent state to other CPUs that started a grace
|
||||
* period. Otherwise we would delay any grace period as long as we run in
|
||||
* the idle task.
|
||||
* Note that if the CPU is in the idle loop from an RCU point of view (ie:
|
||||
* that we are in the section between rcu_idle_enter() and rcu_idle_exit())
|
||||
* then rcu_read_lock_held() sets *ret to false even if the CPU did an
|
||||
* rcu_read_lock(). The reason for this is that RCU ignores CPUs that are
|
||||
* in such a section, considering these as in extended quiescent state,
|
||||
* so such a CPU is effectively never in an RCU read-side critical section
|
||||
* regardless of what RCU primitives it invokes. This state of affairs is
|
||||
* required --- we need to keep an RCU-free window in idle where the CPU may
|
||||
* possibly enter into low power mode. This way we can notice an extended
|
||||
* quiescent state to other CPUs that started a grace period. Otherwise
|
||||
* we would delay any grace period as long as we run in the idle task.
|
||||
*
|
||||
* Similarly, we avoid claiming an SRCU read lock held if the current
|
||||
* Similarly, we avoid claiming an RCU read lock held if the current
|
||||
* CPU is offline.
|
||||
*/
|
||||
static bool rcu_read_lock_held_common(bool *ret)
|
||||
{
|
||||
if (!debug_lockdep_rcu_enabled()) {
|
||||
*ret = 1;
|
||||
return true;
|
||||
}
|
||||
if (!rcu_is_watching()) {
|
||||
*ret = 0;
|
||||
return true;
|
||||
}
|
||||
if (!rcu_lockdep_current_cpu_online()) {
|
||||
*ret = 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int rcu_read_lock_sched_held(void)
|
||||
{
|
||||
int lockdep_opinion = 0;
|
||||
bool ret;
|
||||
|
||||
if (!debug_lockdep_rcu_enabled())
|
||||
return 1;
|
||||
if (!rcu_is_watching())
|
||||
return 0;
|
||||
if (!rcu_lockdep_current_cpu_online())
|
||||
return 0;
|
||||
if (debug_locks)
|
||||
lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
|
||||
return lockdep_opinion || !preemptible();
|
||||
if (rcu_read_lock_held_common(&ret))
|
||||
return ret;
|
||||
return lock_is_held(&rcu_sched_lock_map) || !preemptible();
|
||||
}
|
||||
EXPORT_SYMBOL(rcu_read_lock_sched_held);
|
||||
#endif
|
||||
@ -136,8 +152,7 @@ static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
|
||||
*/
|
||||
bool rcu_gp_is_expedited(void)
|
||||
{
|
||||
return rcu_expedited || atomic_read(&rcu_expedited_nesting) ||
|
||||
rcu_scheduler_active == RCU_SCHEDULER_INIT;
|
||||
return rcu_expedited || atomic_read(&rcu_expedited_nesting);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
|
||||
|
||||
@ -261,12 +276,10 @@ NOKPROBE_SYMBOL(debug_lockdep_rcu_enabled);
|
||||
*/
|
||||
int rcu_read_lock_held(void)
|
||||
{
|
||||
if (!debug_lockdep_rcu_enabled())
|
||||
return 1;
|
||||
if (!rcu_is_watching())
|
||||
return 0;
|
||||
if (!rcu_lockdep_current_cpu_online())
|
||||
return 0;
|
||||
bool ret;
|
||||
|
||||
if (rcu_read_lock_held_common(&ret))
|
||||
return ret;
|
||||
return lock_is_held(&rcu_lock_map);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_read_lock_held);
|
||||
@ -288,16 +301,28 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_held);
|
||||
*/
|
||||
int rcu_read_lock_bh_held(void)
|
||||
{
|
||||
if (!debug_lockdep_rcu_enabled())
|
||||
return 1;
|
||||
if (!rcu_is_watching())
|
||||
return 0;
|
||||
if (!rcu_lockdep_current_cpu_online())
|
||||
return 0;
|
||||
bool ret;
|
||||
|
||||
if (rcu_read_lock_held_common(&ret))
|
||||
return ret;
|
||||
return in_softirq() || irqs_disabled();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
|
||||
|
||||
int rcu_read_lock_any_held(void)
|
||||
{
|
||||
bool ret;
|
||||
|
||||
if (rcu_read_lock_held_common(&ret))
|
||||
return ret;
|
||||
if (lock_is_held(&rcu_lock_map) ||
|
||||
lock_is_held(&rcu_bh_lock_map) ||
|
||||
lock_is_held(&rcu_sched_lock_map))
|
||||
return 1;
|
||||
return !preemptible();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_read_lock_any_held);
|
||||
|
||||
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
||||
|
||||
/**
|
||||
@ -437,6 +462,8 @@ EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_RCU_STALL_COMMON
|
||||
int rcu_cpu_stall_ftrace_dump __read_mostly;
|
||||
module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
|
||||
int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
|
||||
EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
|
||||
module_param(rcu_cpu_stall_suppress, int, 0644);
|
||||
|
@ -3486,8 +3486,36 @@ void scheduler_tick(void)
|
||||
|
||||
struct tick_work {
|
||||
int cpu;
|
||||
atomic_t state;
|
||||
struct delayed_work work;
|
||||
};
|
||||
/* Values for ->state, see diagram below. */
|
||||
#define TICK_SCHED_REMOTE_OFFLINE 0
|
||||
#define TICK_SCHED_REMOTE_OFFLINING 1
|
||||
#define TICK_SCHED_REMOTE_RUNNING 2
|
||||
|
||||
/*
|
||||
* State diagram for ->state:
|
||||
*
|
||||
*
|
||||
* TICK_SCHED_REMOTE_OFFLINE
|
||||
* | ^
|
||||
* | |
|
||||
* | | sched_tick_remote()
|
||||
* | |
|
||||
* | |
|
||||
* +--TICK_SCHED_REMOTE_OFFLINING
|
||||
* | ^
|
||||
* | |
|
||||
* sched_tick_start() | | sched_tick_stop()
|
||||
* | |
|
||||
* V |
|
||||
* TICK_SCHED_REMOTE_RUNNING
|
||||
*
|
||||
*
|
||||
* Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
|
||||
* and sched_tick_start() are happy to leave the state in RUNNING.
|
||||
*/
|
||||
|
||||
static struct tick_work __percpu *tick_work_cpu;
|
||||
|
||||
@ -3500,6 +3528,7 @@ static void sched_tick_remote(struct work_struct *work)
|
||||
struct task_struct *curr;
|
||||
struct rq_flags rf;
|
||||
u64 delta;
|
||||
int os;
|
||||
|
||||
/*
|
||||
* Handle the tick only if it appears the remote CPU is running in full
|
||||
@ -3513,7 +3542,7 @@ static void sched_tick_remote(struct work_struct *work)
|
||||
|
||||
rq_lock_irq(rq, &rf);
|
||||
curr = rq->curr;
|
||||
if (is_idle_task(curr))
|
||||
if (is_idle_task(curr) || cpu_is_offline(cpu))
|
||||
goto out_unlock;
|
||||
|
||||
update_rq_clock(rq);
|
||||
@ -3533,13 +3562,18 @@ out_requeue:
|
||||
/*
|
||||
* Run the remote tick once per second (1Hz). This arbitrary
|
||||
* frequency is large enough to avoid overload but short enough
|
||||
* to keep scheduler internal stats reasonably up to date.
|
||||
* to keep scheduler internal stats reasonably up to date. But
|
||||
* first update state to reflect hotplug activity if required.
|
||||
*/
|
||||
queue_delayed_work(system_unbound_wq, dwork, HZ);
|
||||
os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
|
||||
WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
|
||||
if (os == TICK_SCHED_REMOTE_RUNNING)
|
||||
queue_delayed_work(system_unbound_wq, dwork, HZ);
|
||||
}
|
||||
|
||||
static void sched_tick_start(int cpu)
|
||||
{
|
||||
int os;
|
||||
struct tick_work *twork;
|
||||
|
||||
if (housekeeping_cpu(cpu, HK_FLAG_TICK))
|
||||
@ -3548,15 +3582,20 @@ static void sched_tick_start(int cpu)
|
||||
WARN_ON_ONCE(!tick_work_cpu);
|
||||
|
||||
twork = per_cpu_ptr(tick_work_cpu, cpu);
|
||||
twork->cpu = cpu;
|
||||
INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
|
||||
queue_delayed_work(system_unbound_wq, &twork->work, HZ);
|
||||
os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
|
||||
WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
|
||||
if (os == TICK_SCHED_REMOTE_OFFLINE) {
|
||||
twork->cpu = cpu;
|
||||
INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
|
||||
queue_delayed_work(system_unbound_wq, &twork->work, HZ);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_HOTPLUG_CPU
|
||||
static void sched_tick_stop(int cpu)
|
||||
{
|
||||
struct tick_work *twork;
|
||||
int os;
|
||||
|
||||
if (housekeeping_cpu(cpu, HK_FLAG_TICK))
|
||||
return;
|
||||
@ -3564,7 +3603,10 @@ static void sched_tick_stop(int cpu)
|
||||
WARN_ON_ONCE(!tick_work_cpu);
|
||||
|
||||
twork = per_cpu_ptr(tick_work_cpu, cpu);
|
||||
cancel_delayed_work_sync(&twork->work);
|
||||
/* There cannot be competing actions, but don't rely on stop-machine. */
|
||||
os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
|
||||
WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
|
||||
/* Don't cancel, as this would mess up the state machine. */
|
||||
}
|
||||
#endif /* CONFIG_HOTPLUG_CPU */
|
||||
|
||||
@ -3572,7 +3614,6 @@ int __init sched_tick_offload_init(void)
|
||||
{
|
||||
tick_work_cpu = alloc_percpu(struct tick_work);
|
||||
BUG_ON(!tick_work_cpu);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -241,13 +241,14 @@ static void do_idle(void)
|
||||
check_pgt_cache();
|
||||
rmb();
|
||||
|
||||
local_irq_disable();
|
||||
|
||||
if (cpu_is_offline(cpu)) {
|
||||
tick_nohz_idle_stop_tick_protected();
|
||||
tick_nohz_idle_stop_tick();
|
||||
cpuhp_report_idle_dead();
|
||||
arch_cpu_idle_dead();
|
||||
}
|
||||
|
||||
local_irq_disable();
|
||||
arch_cpu_idle_enter();
|
||||
|
||||
/*
|
||||
|
@ -263,7 +263,6 @@ static void torture_onoff_cleanup(void)
|
||||
onoff_task = NULL;
|
||||
#endif /* #ifdef CONFIG_HOTPLUG_CPU */
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(torture_onoff_cleanup);
|
||||
|
||||
/*
|
||||
* Print online/offline testing statistics.
|
||||
@ -449,7 +448,6 @@ static void torture_shuffle_cleanup(void)
|
||||
}
|
||||
shuffler_task = NULL;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(torture_shuffle_cleanup);
|
||||
|
||||
/*
|
||||
* Variables for auto-shutdown. This allows "lights out" torture runs
|
||||
|
@ -6,22 +6,22 @@
|
||||
|
||||
/*
|
||||
* Traverse the ftrace_global_list, invoking all entries. The reason that we
|
||||
* can use rcu_dereference_raw_notrace() is that elements removed from this list
|
||||
* can use rcu_dereference_raw_check() is that elements removed from this list
|
||||
* are simply leaked, so there is no need to interact with a grace-period
|
||||
* mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
|
||||
* mechanism. The rcu_dereference_raw_check() calls are needed to handle
|
||||
* concurrent insertions into the ftrace_global_list.
|
||||
*
|
||||
* Silly Alpha and silly pointer-speculation compiler optimizations!
|
||||
*/
|
||||
#define do_for_each_ftrace_op(op, list) \
|
||||
op = rcu_dereference_raw_notrace(list); \
|
||||
op = rcu_dereference_raw_check(list); \
|
||||
do
|
||||
|
||||
/*
|
||||
* Optimized for just a single item in the list (as that is the normal case).
|
||||
*/
|
||||
#define while_for_each_ftrace_op(op) \
|
||||
while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
|
||||
while (likely(op = rcu_dereference_raw_check((op)->next)) && \
|
||||
unlikely((op) != &ftrace_list_end))
|
||||
|
||||
extern struct ftrace_ops __rcu *ftrace_ops_list;
|
||||
|
@ -2642,10 +2642,10 @@ static void ftrace_exports(struct ring_buffer_event *event)
|
||||
|
||||
preempt_disable_notrace();
|
||||
|
||||
export = rcu_dereference_raw_notrace(ftrace_exports_list);
|
||||
export = rcu_dereference_raw_check(ftrace_exports_list);
|
||||
while (export) {
|
||||
trace_process_export(export, event);
|
||||
export = rcu_dereference_raw_notrace(export->next);
|
||||
export = rcu_dereference_raw_check(export->next);
|
||||
}
|
||||
|
||||
preempt_enable_notrace();
|
||||
|
@ -124,7 +124,8 @@ struct fib_table *fib_get_table(struct net *net, u32 id)
|
||||
h = id & (FIB_TABLE_HASHSZ - 1);
|
||||
|
||||
head = &net->ipv4.fib_table_hash[h];
|
||||
hlist_for_each_entry_rcu(tb, head, tb_hlist) {
|
||||
hlist_for_each_entry_rcu(tb, head, tb_hlist,
|
||||
lockdep_rtnl_is_held()) {
|
||||
if (tb->tb_id == id)
|
||||
return tb;
|
||||
}
|
||||
|
@ -227,7 +227,7 @@ then
|
||||
must_continue=yes
|
||||
fi
|
||||
last_ts="`tail $resdir/console.log | grep '^\[ *[0-9]\+\.[0-9]\+]' | tail -1 | sed -e 's/^\[ *//' -e 's/\..*$//'`"
|
||||
if test -z "last_ts"
|
||||
if test -z "$last_ts"
|
||||
then
|
||||
last_ts=0
|
||||
fi
|
||||
|
@ -3,3 +3,4 @@ rcutree.gp_preinit_delay=12
|
||||
rcutree.gp_init_delay=3
|
||||
rcutree.gp_cleanup_delay=3
|
||||
rcutree.kthread_prio=2
|
||||
threadirqs
|
||||
|
Loading…
Reference in New Issue
Block a user