mirror of
https://github.com/torvalds/linux.git
synced 2024-12-10 05:01:26 +00:00
9c61e78954
The callback and the irq work are never used at the same time. Putting them into an union saves us 24 bytes and makes the structure only 120 bytes in size. Signed-off-by: Christian König <christian.koenig@amd.com> Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch> Link: https://patchwork.freedesktop.org/patch/msgid/20210611120301.10595-2-christian.koenig@amd.com
259 lines
6.6 KiB
C
259 lines
6.6 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* fence-chain: chain fences together in a timeline
|
|
*
|
|
* Copyright (C) 2018 Advanced Micro Devices, Inc.
|
|
* Authors:
|
|
* Christian König <christian.koenig@amd.com>
|
|
*/
|
|
|
|
#include <linux/dma-fence-chain.h>
|
|
|
|
static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
|
|
|
|
/**
|
|
* dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
|
|
* @chain: chain node to get the previous node from
|
|
*
|
|
* Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
|
|
* chain node.
|
|
*/
|
|
static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
|
|
{
|
|
struct dma_fence *prev;
|
|
|
|
rcu_read_lock();
|
|
prev = dma_fence_get_rcu_safe(&chain->prev);
|
|
rcu_read_unlock();
|
|
return prev;
|
|
}
|
|
|
|
/**
|
|
* dma_fence_chain_walk - chain walking function
|
|
* @fence: current chain node
|
|
*
|
|
* Walk the chain to the next node. Returns the next fence or NULL if we are at
|
|
* the end of the chain. Garbage collects chain nodes which are already
|
|
* signaled.
|
|
*/
|
|
struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
|
|
{
|
|
struct dma_fence_chain *chain, *prev_chain;
|
|
struct dma_fence *prev, *replacement, *tmp;
|
|
|
|
chain = to_dma_fence_chain(fence);
|
|
if (!chain) {
|
|
dma_fence_put(fence);
|
|
return NULL;
|
|
}
|
|
|
|
while ((prev = dma_fence_chain_get_prev(chain))) {
|
|
|
|
prev_chain = to_dma_fence_chain(prev);
|
|
if (prev_chain) {
|
|
if (!dma_fence_is_signaled(prev_chain->fence))
|
|
break;
|
|
|
|
replacement = dma_fence_chain_get_prev(prev_chain);
|
|
} else {
|
|
if (!dma_fence_is_signaled(prev))
|
|
break;
|
|
|
|
replacement = NULL;
|
|
}
|
|
|
|
tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
|
|
prev, replacement);
|
|
if (tmp == prev)
|
|
dma_fence_put(tmp);
|
|
else
|
|
dma_fence_put(replacement);
|
|
dma_fence_put(prev);
|
|
}
|
|
|
|
dma_fence_put(fence);
|
|
return prev;
|
|
}
|
|
EXPORT_SYMBOL(dma_fence_chain_walk);
|
|
|
|
/**
|
|
* dma_fence_chain_find_seqno - find fence chain node by seqno
|
|
* @pfence: pointer to the chain node where to start
|
|
* @seqno: the sequence number to search for
|
|
*
|
|
* Advance the fence pointer to the chain node which will signal this sequence
|
|
* number. If no sequence number is provided then this is a no-op.
|
|
*
|
|
* Returns EINVAL if the fence is not a chain node or the sequence number has
|
|
* not yet advanced far enough.
|
|
*/
|
|
int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
|
|
{
|
|
struct dma_fence_chain *chain;
|
|
|
|
if (!seqno)
|
|
return 0;
|
|
|
|
chain = to_dma_fence_chain(*pfence);
|
|
if (!chain || chain->base.seqno < seqno)
|
|
return -EINVAL;
|
|
|
|
dma_fence_chain_for_each(*pfence, &chain->base) {
|
|
if ((*pfence)->context != chain->base.context ||
|
|
to_dma_fence_chain(*pfence)->prev_seqno < seqno)
|
|
break;
|
|
}
|
|
dma_fence_put(&chain->base);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(dma_fence_chain_find_seqno);
|
|
|
|
static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
|
|
{
|
|
return "dma_fence_chain";
|
|
}
|
|
|
|
static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
|
|
{
|
|
return "unbound";
|
|
}
|
|
|
|
static void dma_fence_chain_irq_work(struct irq_work *work)
|
|
{
|
|
struct dma_fence_chain *chain;
|
|
|
|
chain = container_of(work, typeof(*chain), work);
|
|
|
|
/* Try to rearm the callback */
|
|
if (!dma_fence_chain_enable_signaling(&chain->base))
|
|
/* Ok, we are done. No more unsignaled fences left */
|
|
dma_fence_signal(&chain->base);
|
|
dma_fence_put(&chain->base);
|
|
}
|
|
|
|
static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
|
|
{
|
|
struct dma_fence_chain *chain;
|
|
|
|
chain = container_of(cb, typeof(*chain), cb);
|
|
init_irq_work(&chain->work, dma_fence_chain_irq_work);
|
|
irq_work_queue(&chain->work);
|
|
dma_fence_put(f);
|
|
}
|
|
|
|
static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
|
|
{
|
|
struct dma_fence_chain *head = to_dma_fence_chain(fence);
|
|
|
|
dma_fence_get(&head->base);
|
|
dma_fence_chain_for_each(fence, &head->base) {
|
|
struct dma_fence_chain *chain = to_dma_fence_chain(fence);
|
|
struct dma_fence *f = chain ? chain->fence : fence;
|
|
|
|
dma_fence_get(f);
|
|
if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
|
|
dma_fence_put(fence);
|
|
return true;
|
|
}
|
|
dma_fence_put(f);
|
|
}
|
|
dma_fence_put(&head->base);
|
|
return false;
|
|
}
|
|
|
|
static bool dma_fence_chain_signaled(struct dma_fence *fence)
|
|
{
|
|
dma_fence_chain_for_each(fence, fence) {
|
|
struct dma_fence_chain *chain = to_dma_fence_chain(fence);
|
|
struct dma_fence *f = chain ? chain->fence : fence;
|
|
|
|
if (!dma_fence_is_signaled(f)) {
|
|
dma_fence_put(fence);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
static void dma_fence_chain_release(struct dma_fence *fence)
|
|
{
|
|
struct dma_fence_chain *chain = to_dma_fence_chain(fence);
|
|
struct dma_fence *prev;
|
|
|
|
/* Manually unlink the chain as much as possible to avoid recursion
|
|
* and potential stack overflow.
|
|
*/
|
|
while ((prev = rcu_dereference_protected(chain->prev, true))) {
|
|
struct dma_fence_chain *prev_chain;
|
|
|
|
if (kref_read(&prev->refcount) > 1)
|
|
break;
|
|
|
|
prev_chain = to_dma_fence_chain(prev);
|
|
if (!prev_chain)
|
|
break;
|
|
|
|
/* No need for atomic operations since we hold the last
|
|
* reference to prev_chain.
|
|
*/
|
|
chain->prev = prev_chain->prev;
|
|
RCU_INIT_POINTER(prev_chain->prev, NULL);
|
|
dma_fence_put(prev);
|
|
}
|
|
dma_fence_put(prev);
|
|
|
|
dma_fence_put(chain->fence);
|
|
dma_fence_free(fence);
|
|
}
|
|
|
|
const struct dma_fence_ops dma_fence_chain_ops = {
|
|
.use_64bit_seqno = true,
|
|
.get_driver_name = dma_fence_chain_get_driver_name,
|
|
.get_timeline_name = dma_fence_chain_get_timeline_name,
|
|
.enable_signaling = dma_fence_chain_enable_signaling,
|
|
.signaled = dma_fence_chain_signaled,
|
|
.release = dma_fence_chain_release,
|
|
};
|
|
EXPORT_SYMBOL(dma_fence_chain_ops);
|
|
|
|
/**
|
|
* dma_fence_chain_init - initialize a fence chain
|
|
* @chain: the chain node to initialize
|
|
* @prev: the previous fence
|
|
* @fence: the current fence
|
|
* @seqno: the sequence number to use for the fence chain
|
|
*
|
|
* Initialize a new chain node and either start a new chain or add the node to
|
|
* the existing chain of the previous fence.
|
|
*/
|
|
void dma_fence_chain_init(struct dma_fence_chain *chain,
|
|
struct dma_fence *prev,
|
|
struct dma_fence *fence,
|
|
uint64_t seqno)
|
|
{
|
|
struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
|
|
uint64_t context;
|
|
|
|
spin_lock_init(&chain->lock);
|
|
rcu_assign_pointer(chain->prev, prev);
|
|
chain->fence = fence;
|
|
chain->prev_seqno = 0;
|
|
|
|
/* Try to reuse the context of the previous chain node. */
|
|
if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
|
|
context = prev->context;
|
|
chain->prev_seqno = prev->seqno;
|
|
} else {
|
|
context = dma_fence_context_alloc(1);
|
|
/* Make sure that we always have a valid sequence number. */
|
|
if (prev_chain)
|
|
seqno = max(prev->seqno, seqno);
|
|
}
|
|
|
|
dma_fence_init(&chain->base, &dma_fence_chain_ops,
|
|
&chain->lock, context, seqno);
|
|
}
|
|
EXPORT_SYMBOL(dma_fence_chain_init);
|