Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-21 19:41:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

bcachefs: rcu_pending now works in userspace

Browse Source 
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
This commit is contained in:
Kent Overstreet 2024-08-23 18:21:31 -04:00
parent 8e973a4f3c
commit d2ed0f206a
2 changed files with 53 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
fs/bcachefs/rcu_pending.c
View File

@ -115,24 +115,40 @@ static bool __rcu_pending_has_pending(struct rcu_pending_pcpu *p)
static void rcu_pending_list_merge(struct rcu_pending_list *l1, static void rcu_pending_list_merge(struct rcu_pending_list *l1,
struct rcu_pending_list *l2) struct rcu_pending_list *l2)
{ {
#ifdef __KERNEL__
if (!l1->head) if (!l1->head)
l1->head = l2->head; l1->head = l2->head;
else else
l1->tail->next = l2->head; l1->tail->next = l2->head;
l1->tail = l2->tail; #else
if (!l1->head)
l1->head = l2->head;
else
l1->tail->next.next = (void *) l2->head;
#endif
l1->tail = l2->tail;
l2->head = l2->tail = NULL; l2->head = l2->tail = NULL;
} }
static void rcu_pending_list_add(struct rcu_pending_list *l, static void rcu_pending_list_add(struct rcu_pending_list *l,
struct rcu_head *n) struct rcu_head *n)
{ {
#ifdef __KERNEL__
if (!l->head) if (!l->head)
l->head = n; l->head = n;
else else
l->tail->next = n; l->tail->next = n;
l->tail = n; l->tail = n;
n->next = NULL; n->next = NULL;
#else
if (!l->head)
l->head = n;
else
l->tail->next.next = (void *) n;
l->tail = n;
n->next.next = NULL;
#endif
} }
static void merge_expired_lists(struct rcu_pending_pcpu *p) static void merge_expired_lists(struct rcu_pending_pcpu *p)
@ -144,6 +160,19 @@ static void merge_expired_lists(struct rcu_pending_pcpu *p)
rcu_pending_list_merge(expired, i); rcu_pending_list_merge(expired, i);
} }
#ifndef __KERNEL__
static inline void kfree_bulk(size_t nr, void ** p)
{
while (nr--)
kfree(*p);
}
#define local_irq_save(flags) \
do { \
flags = 0; \
} while (0)
#endif
static noinline void __process_finished_items(struct rcu_pending *pending, static noinline void __process_finished_items(struct rcu_pending *pending,
struct rcu_pending_pcpu *p, struct rcu_pending_pcpu *p,
unsigned long flags) unsigned long flags)
@ -177,7 +206,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending,
while (list) { while (list) {
struct rcu_head *obj = list; struct rcu_head *obj = list;
#ifdef __KERNEL__
list = obj->next; list = obj->next;
#else
list = (void *) obj->next.next;
#endif
/* /*
* low bit of pointer indicates whether rcu_head needs * low bit of pointer indicates whether rcu_head needs
@ -204,7 +237,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending,
while (list) { while (list) {
struct rcu_head *obj = list; struct rcu_head *obj = list;
#ifdef __KERNEL__
list = obj->next; list = obj->next;
#else
list = (void *) obj->next.next;
#endif
obj->func(obj); obj->func(obj);
} }
break; break;
@ -216,7 +253,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending,
while (list) { while (list) {
struct rcu_head *obj = list; struct rcu_head *obj = list;
#ifdef __KERNEL__
list = obj->next; list = obj->next;
#else
list = (void *) obj->next.next;
#endif
pending->process(pending, obj); pending->process(pending, obj);
} }
break; break;
@ -265,11 +306,13 @@ static void rcu_pending_rcu_cb(struct rcu_head *rcu)
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&p->lock, flags); spin_lock_irqsave(&p->lock, flags);
if (__rcu_pending_has_pending(p)) if (__rcu_pending_has_pending(p)) {
spin_unlock_irqrestore(&p->lock, flags);
__call_rcu(p->parent->srcu, &p->cb, rcu_pending_rcu_cb); __call_rcu(p->parent->srcu, &p->cb, rcu_pending_rcu_cb);
else } else {
p->cb_armed = false; p->cb_armed = false;
spin_unlock_irqrestore(&p->lock, flags); spin_unlock_irqrestore(&p->lock, flags);
}
} }
static __always_inline struct rcu_pending_seq * static __always_inline struct rcu_pending_seq *
@ -489,7 +532,11 @@ static struct rcu_head *rcu_pending_pcpu_dequeue(struct rcu_pending_pcpu *p)
static_array_for_each(p->lists, i) static_array_for_each(p->lists, i)
if (i->head) { if (i->head) {
ret = i->head; ret = i->head;
#ifdef __KERNEL__
i->head = ret->next; i->head = ret->next;
#else
i->head = (void *) ret->next.next;
#endif
if (!i->head) if (!i->head)
i->tail = NULL; i->tail = NULL;
goto out; goto out;

2
fs/bcachefs/rcu_pending.h
View File

@ -2,6 +2,8 @@
#ifndef _LINUX_RCU_PENDING_H #ifndef _LINUX_RCU_PENDING_H
#define _LINUX_RCU_PENDING_H #define _LINUX_RCU_PENDING_H
#include <linux/rcupdate.h>
struct rcu_pending; struct rcu_pending;
typedef void (*rcu_pending_process_fn)(struct rcu_pending *, struct rcu_head *); typedef void (*rcu_pending_process_fn)(struct rcu_pending *, struct rcu_head *);