leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

task_work: use try_cmpxchg in task_work_add, task_work_cancel_match and task_work_run

Browse Source 
Use try_cmpxchg instead of cmpxchg (*ptr, old, new) == old in
task_work_add, task_work_cancel_match and task_work_run.  x86 CMPXCHG
instruction returns success in ZF flag, so this change saves a compare
after cmpxchg (and related move instruction in front of cmpxchg).

Also, atomic_try_cmpxchg implicitly assigns old *ptr value to "old"
when cmpxchg fails, enabling further code simplifications.

The patch avoids extra memory read in case cmpxchg fails.

Link: https://lkml.kernel.org/r/20220823152632.4517-1-ubizjak@gmail.com
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Uros Bizjak 2022-08-23 17:26:32 +02:00 committed by Andrew Morton
parent 977bbf4385
commit 5fdfa161b2
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
kernel/task_work.c
View File

@ -47,12 +47,12 @@ int task_work_add(struct task_struct *task, struct callback_head *work,
/* record the work call stack in order to print it in KASAN reports */ /* record the work call stack in order to print it in KASAN reports */
kasan_record_aux_stack(work); kasan_record_aux_stack(work);
do {
head = READ_ONCE(task->task_works); head = READ_ONCE(task->task_works);
do {
if (unlikely(head == &work_exited)) if (unlikely(head == &work_exited))
return -ESRCH; return -ESRCH;
work->next = head; work->next = head;
} while (cmpxchg(&task->task_works, head, work) != head); } while (!try_cmpxchg(&task->task_works, &head, work));
switch (notify) { switch (notify) {
case TWA_NONE: case TWA_NONE:
@ -100,10 +100,12 @@ task_work_cancel_match(struct task_struct *task,
* we raced with task_work_run(), *pprev == NULL/exited. * we raced with task_work_run(), *pprev == NULL/exited.
*/ */
raw_spin_lock_irqsave(&task->pi_lock, flags); raw_spin_lock_irqsave(&task->pi_lock, flags);
while ((work = READ_ONCE(*pprev))) { work = READ_ONCE(*pprev);
if (!match(work, data)) while (work) {
if (!match(work, data)) {
pprev = &work->next; pprev = &work->next;
else if (cmpxchg(pprev, work, work->next) == work) work = READ_ONCE(*pprev);
} else if (try_cmpxchg(pprev, &work, work->next))
break; break;
} }
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock_irqrestore(&task->pi_lock, flags);
@ -151,16 +153,16 @@ void task_work_run(void)
* work->func() can do task_work_add(), do not set * work->func() can do task_work_add(), do not set
* work_exited unless the list is empty. * work_exited unless the list is empty.
*/ */
work = READ_ONCE(task->task_works);
do { do {
head = NULL; head = NULL;
work = READ_ONCE(task->task_works);
if (!work) { if (!work) {
if (task->flags & PF_EXITING) if (task->flags & PF_EXITING)
head = &work_exited; head = &work_exited;
else else
break; break;
} }
} while (cmpxchg(&task->task_works, work, head) != work); } while (!try_cmpxchg(&task->task_works, &work, head));
if (!work) if (!work)
break; break;