Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-24 21:21:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

srcu: Unconditionally record srcu_read_lock_lite() in ->srcu_reader_flavor

Browse Source 
Currently, srcu_read_lock_lite() uses the SRCU_READ_FLAVOR_LITE bit in
->srcu_reader_flavor to communicate to the grace-period processing in
srcu_readers_active_idx_check() that the smp_mb() must be replaced by a
synchronize_rcu().  Unfortunately, ->srcu_reader_flavor is not updated
unless the kernel is built with CONFIG_PROVE_RCU=y.  Therefore in all
kernels built with CONFIG_PROVE_RCU=n, srcu_readers_active_idx_check()
incorrectly uses smp_mb() instead of synchronize_rcu() for srcu_struct
structures whose readers use srcu_read_lock_lite().

This commit therefore causes Tree SRCU srcu_read_lock_lite()
to unconditionally update ->srcu_reader_flavor so that
srcu_readers_active_idx_check() can make the correct choice.

Reported-by: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com>
Closes: https://lore.kernel.org/all/d07e8f4a-d5ff-4c8e-8e61-50db285c57e9@amd.com/
Fixes: c0f08d6b5a61 ("srcu: Add srcu_read_lock_lite() and srcu_read_unlock_lite()")
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Frederic Weisbecker <frederic@kernel.org>
Reviewed-by: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
This commit is contained in:
Paul E. McKenney 2024-11-11 16:28:51 -08:00 committed by Frederic Weisbecker
parent f8ce622ac9
commit 9407f5c3ec
4 changed files with 27 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/linux/srcu.h
View File

@ -183,12 +183,6 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_TREE_SRCU)
void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);
#else
#define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
#endif
/**
* srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
@ -277,7 +271,7 @@ static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp)
{
int retval;
srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_LITE);
srcu_check_read_flavor_lite(ssp);
retval = __srcu_read_lock_lite(ssp);
rcu_try_lock_acquire(&ssp->dep_map);
return retval;

3
include/linux/srcutiny.h
View File

@ -81,6 +81,9 @@ static inline void srcu_barrier(struct srcu_struct *ssp)
synchronize_srcu(ssp);
}
#define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
#define srcu_check_read_flavor_lite(ssp) do { } while (0)
/* Defined here to avoid size increase for non-torture kernels. */
static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
char *tt, char *tf)

21
include/linux/srcutree.h
View File

@ -248,4 +248,25 @@ static inline void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)
RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_lite().");
}
void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);
// Record _lite() usage even for CONFIG_PROVE_RCU=n kernels.
static inline void srcu_check_read_flavor_lite(struct srcu_struct *ssp)
{
struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);
if (likely(READ_ONCE(sdp->srcu_reader_flavor) & SRCU_READ_FLAVOR_LITE))
return;
// Note that the cmpxchg() in srcu_check_read_flavor() is fully ordered.
__srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_LITE);
}
// Record non-_lite() usage only for CONFIG_PROVE_RCU=y kernels.
static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
{
if (IS_ENABLED(CONFIG_PROVE_RCU))
__srcu_check_read_flavor(ssp, read_flavor);
}
#endif

6
kernel/rcu/srcutree.c
View File

@ -712,11 +712,10 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
}
EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
#ifdef CONFIG_PROVE_RCU
/*
* Check for consistent reader flavor.
*/
void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
{
int old_read_flavor;
struct srcu_data *sdp;
@ -734,8 +733,7 @@ void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
}
WARN_ONCE(old_read_flavor != read_flavor, "CPU %d old state %d new state %d\n", sdp->cpu, old_read_flavor, read_flavor);
}
EXPORT_SYMBOL_GPL(srcu_check_read_flavor);
#endif /* CONFIG_PROVE_RCU */
EXPORT_SYMBOL_GPL(__srcu_check_read_flavor);
/*
* Counts the new reader in the appropriate per-CPU element of the