From 825dbc6ff7a3a063ea91be7d94af940080b0c991 Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Tue, 15 Oct 2019 11:26:15 +0100 Subject: [PATCH 1/3] percpu: add __percpu to SHIFT_PERCPU_PTR The SHIFT_PERCPU_PTR() returns a pointer used by a number of functions that expect the pointer to be __percpu annotated (sparse address space 3). Adding __percpu to this makes the following sparse warnings go away. Note, this then creates the problem the __percup is marked as noderef, which may need removing for some of the internal functions, or to remove other warnings. mm/vmstat.c:385:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:385:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:385:13: got signed char * mm/vmstat.c:385:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:385:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:385:13: got signed char * mm/vmstat.c:385:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:385:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:385:13: got signed char * mm/vmstat.c:385:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:385:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:385:13: got signed char * mm/vmstat.c:401:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:401:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:401:13: got signed char * mm/vmstat.c:401:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:401:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:401:13: got signed char * mm/vmstat.c:401:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:401:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:401:13: got signed char * mm/vmstat.c:401:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:401:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:401:13: got signed char * mm/vmstat.c:429:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:429:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:429:13: got signed char * mm/vmstat.c:429:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:429:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:429:13: got signed char * mm/vmstat.c:429:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:429:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:429:13: got signed char * mm/vmstat.c:429:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:429:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:429:13: got signed char * mm/vmstat.c:445:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:445:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:445:13: got signed char * mm/vmstat.c:445:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:445:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:445:13: got signed char * mm/vmstat.c:445:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:445:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:445:13: got signed char * mm/vmstat.c:445:13: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:445:13: expected signed char [noderef] [usertype] *__p mm/vmstat.c:445:13: got signed char * mm/vmstat.c:763:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:763:29: expected signed char [noderef] *__p mm/vmstat.c:763:29: got signed char * mm/vmstat.c:763:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:763:29: expected signed char [noderef] *__p mm/vmstat.c:763:29: got signed char * mm/vmstat.c:763:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:763:29: expected signed char [noderef] *__p mm/vmstat.c:763:29: got signed char * mm/vmstat.c:763:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:763:29: expected signed char [noderef] *__p mm/vmstat.c:763:29: got signed char * mm/vmstat.c:825:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:825:29: expected signed char [noderef] *__p mm/vmstat.c:825:29: got signed char * mm/vmstat.c:825:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:825:29: expected signed char [noderef] *__p mm/vmstat.c:825:29: got signed char * mm/vmstat.c:825:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:825:29: expected signed char [noderef] *__p mm/vmstat.c:825:29: got signed char * mm/vmstat.c:825:29: warning: incorrect type in initializer (different address spaces) mm/vmstat.c:825:29: expected signed char [noderef] *__p mm/vmstat.c:825:29: got signed char * Signed-off-by: Ben Dooks Signed-off-by: Dennis Zhou --- include/linux/percpu-defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index a6fabd865211..a49b6c702598 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -229,7 +229,7 @@ do { \ * pointer value. The weird cast keeps both GCC and sparse happy. */ #define SHIFT_PERCPU_PTR(__p, __offset) \ - RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)) + RELOC_HIDE((typeof(*(__p)) __kernel __percpu __force *)(__p), (__offset)) #define per_cpu_ptr(ptr, cpu) \ ({ \ From 9e8d42a0f7eb9056f8bdb241b91738b5a2923f4c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 8 Nov 2019 18:35:53 +0100 Subject: [PATCH 2/3] percpu-refcount: Use normal instead of RCU-sched" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a revert of commit a4244454df129 ("percpu-refcount: use RCU-sched insted of normal RCU") which claims the only reason for using RCU-sched is "rcu_read_[un]lock() … are slightly more expensive than preempt_disable/enable()" and "As the RCU critical sections are extremely short, using sched-RCU shouldn't have any latency implications." The problem with using RCU-sched here is that it disables preemption and the release callback (called from percpu_ref_put_many()) must not acquire any sleeping locks like spinlock_t. This breaks PREEMPT_RT because some of the users acquire spinlock_t locks in their callbacks. Using rcu_read_lock() on PREEMPTION=n kernels is not any different compared to rcu_read_lock_sched(). On PREEMPTION=y kernels there are already performance issues due to additional preemption points. Looking at the code, the rcu_read_lock() is just an increment and unlock is almost just a decrement unless there is something special to do. Both are functions while disabling preemption is inlined. Doing a small benchmark, the minimal amount of time required was mostly the same. The average time required was higher due to the higher MAX value (which could be preemption). With DEBUG_PREEMPT=y it is rcu_read_lock_sched() that takes a little longer due to the additional debug code. Convert back to normal RCU. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Dennis Zhou --- include/linux/percpu-refcount.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index 7aef0abc194a..390031e816dc 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -186,14 +186,14 @@ static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr) { unsigned long __percpu *percpu_count; - rcu_read_lock_sched(); + rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) this_cpu_add(*percpu_count, nr); else atomic_long_add(nr, &ref->count); - rcu_read_unlock_sched(); + rcu_read_unlock(); } /** @@ -223,7 +223,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) unsigned long __percpu *percpu_count; bool ret; - rcu_read_lock_sched(); + rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) { this_cpu_inc(*percpu_count); @@ -232,7 +232,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) ret = atomic_long_inc_not_zero(&ref->count); } - rcu_read_unlock_sched(); + rcu_read_unlock(); return ret; } @@ -257,7 +257,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) unsigned long __percpu *percpu_count; bool ret = false; - rcu_read_lock_sched(); + rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) { this_cpu_inc(*percpu_count); @@ -266,7 +266,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) ret = atomic_long_inc_not_zero(&ref->count); } - rcu_read_unlock_sched(); + rcu_read_unlock(); return ret; } @@ -285,14 +285,14 @@ static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr) { unsigned long __percpu *percpu_count; - rcu_read_lock_sched(); + rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) this_cpu_sub(*percpu_count, nr); else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) ref->release(ref); - rcu_read_unlock_sched(); + rcu_read_unlock(); } /** From ba30e27405afa0b13b79532a345977b3e58ad501 Mon Sep 17 00:00:00 2001 From: Dennis Zhou Date: Mon, 25 Nov 2019 14:28:04 -0800 Subject: [PATCH 3/3] Revert "percpu: add __percpu to SHIFT_PERCPU_PTR" This reverts commit 825dbc6ff7a3a063ea91be7d94af940080b0c991. I mistakenly applied this and only now have thought about it a little more and had time to evaluate a kbuild error for dmaengine. Once we're calling RELOC_HIDE, we're moving back into the __kernel address space and letting users interact with the actual memory address rather than in __percpu which is before adding the offsets. Signed-off-by: Dennis Zhou --- include/linux/percpu-defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index a49b6c702598..a6fabd865211 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -229,7 +229,7 @@ do { \ * pointer value. The weird cast keeps both GCC and sparse happy. */ #define SHIFT_PERCPU_PTR(__p, __offset) \ - RELOC_HIDE((typeof(*(__p)) __kernel __percpu __force *)(__p), (__offset)) + RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)) #define per_cpu_ptr(ptr, cpu) \ ({ \