Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * fence-chain: chain fences together in a timeline
4 *
5 * Copyright (C) 2018 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
8 */
9
10#include <linux/dma-fence-chain.h>
11
12static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
13
14/**
15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
16 * @chain: chain node to get the previous node from
17 *
18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
19 * chain node.
20 */
21static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
22{
23 struct dma_fence *prev;
24
25 rcu_read_lock();
26 prev = dma_fence_get_rcu_safe(&chain->prev);
27 rcu_read_unlock();
28 return prev;
29}
30
31/**
32 * dma_fence_chain_walk - chain walking function
33 * @fence: current chain node
34 *
35 * Walk the chain to the next node. Returns the next fence or NULL if we are at
36 * the end of the chain. Garbage collects chain nodes which are already
37 * signaled.
38 */
39struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
40{
41 struct dma_fence_chain *chain, *prev_chain;
42 struct dma_fence *prev, *replacement, *tmp;
43
44 chain = to_dma_fence_chain(fence);
45 if (!chain) {
46 dma_fence_put(fence);
47 return NULL;
48 }
49
50 while ((prev = dma_fence_chain_get_prev(chain))) {
51
52 prev_chain = to_dma_fence_chain(prev);
53 if (prev_chain) {
54 if (!dma_fence_is_signaled(prev_chain->fence))
55 break;
56
57 replacement = dma_fence_chain_get_prev(prev_chain);
58 } else {
59 if (!dma_fence_is_signaled(prev))
60 break;
61
62 replacement = NULL;
63 }
64
65 tmp = unrcu_pointer(cmpxchg(&chain->prev, RCU_INITIALIZER(prev),
66 RCU_INITIALIZER(replacement)));
67 if (tmp == prev)
68 dma_fence_put(tmp);
69 else
70 dma_fence_put(replacement);
71 dma_fence_put(prev);
72 }
73
74 dma_fence_put(fence);
75 return prev;
76}
77EXPORT_SYMBOL(dma_fence_chain_walk);
78
79/**
80 * dma_fence_chain_find_seqno - find fence chain node by seqno
81 * @pfence: pointer to the chain node where to start
82 * @seqno: the sequence number to search for
83 *
84 * Advance the fence pointer to the chain node which will signal this sequence
85 * number. If no sequence number is provided then this is a no-op.
86 *
87 * Returns EINVAL if the fence is not a chain node or the sequence number has
88 * not yet advanced far enough.
89 */
90int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
91{
92 struct dma_fence_chain *chain;
93
94 if (!seqno)
95 return 0;
96
97 chain = to_dma_fence_chain(*pfence);
98 if (!chain || chain->base.seqno < seqno)
99 return -EINVAL;
100
101 dma_fence_chain_for_each(*pfence, &chain->base) {
102 if ((*pfence)->context != chain->base.context ||
103 to_dma_fence_chain(*pfence)->prev_seqno < seqno)
104 break;
105 }
106 dma_fence_put(&chain->base);
107
108 return 0;
109}
110EXPORT_SYMBOL(dma_fence_chain_find_seqno);
111
112static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
113{
114 return "dma_fence_chain";
115}
116
117static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
118{
119 return "unbound";
120}
121
122static void dma_fence_chain_irq_work(struct irq_work *work)
123{
124 struct dma_fence_chain *chain;
125
126 chain = container_of(work, typeof(*chain), work);
127
128 /* Try to rearm the callback */
129 if (!dma_fence_chain_enable_signaling(&chain->base))
130 /* Ok, we are done. No more unsignaled fences left */
131 dma_fence_signal(&chain->base);
132 dma_fence_put(&chain->base);
133}
134
135static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
136{
137 struct dma_fence_chain *chain;
138
139 chain = container_of(cb, typeof(*chain), cb);
140 init_irq_work(&chain->work, dma_fence_chain_irq_work);
141 irq_work_queue(&chain->work);
142 dma_fence_put(f);
143}
144
145static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
146{
147 struct dma_fence_chain *head = to_dma_fence_chain(fence);
148
149 dma_fence_get(&head->base);
150 dma_fence_chain_for_each(fence, &head->base) {
151 struct dma_fence *f = dma_fence_chain_contained(fence);
152
153 dma_fence_get(f);
154 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
155 dma_fence_put(fence);
156 return true;
157 }
158 dma_fence_put(f);
159 }
160 dma_fence_put(&head->base);
161 return false;
162}
163
164static bool dma_fence_chain_signaled(struct dma_fence *fence)
165{
166 dma_fence_chain_for_each(fence, fence) {
167 struct dma_fence *f = dma_fence_chain_contained(fence);
168
169 if (!dma_fence_is_signaled(f)) {
170 dma_fence_put(fence);
171 return false;
172 }
173 }
174
175 return true;
176}
177
178static void dma_fence_chain_release(struct dma_fence *fence)
179{
180 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
181 struct dma_fence *prev;
182
183 /* Manually unlink the chain as much as possible to avoid recursion
184 * and potential stack overflow.
185 */
186 while ((prev = rcu_dereference_protected(chain->prev, true))) {
187 struct dma_fence_chain *prev_chain;
188
189 if (kref_read(&prev->refcount) > 1)
190 break;
191
192 prev_chain = to_dma_fence_chain(prev);
193 if (!prev_chain)
194 break;
195
196 /* No need for atomic operations since we hold the last
197 * reference to prev_chain.
198 */
199 chain->prev = prev_chain->prev;
200 RCU_INIT_POINTER(prev_chain->prev, NULL);
201 dma_fence_put(prev);
202 }
203 dma_fence_put(prev);
204
205 dma_fence_put(chain->fence);
206 dma_fence_free(fence);
207}
208
209
210static void dma_fence_chain_set_deadline(struct dma_fence *fence,
211 ktime_t deadline)
212{
213 dma_fence_chain_for_each(fence, fence) {
214 struct dma_fence *f = dma_fence_chain_contained(fence);
215
216 dma_fence_set_deadline(f, deadline);
217 }
218}
219
220const struct dma_fence_ops dma_fence_chain_ops = {
221 .use_64bit_seqno = true,
222 .get_driver_name = dma_fence_chain_get_driver_name,
223 .get_timeline_name = dma_fence_chain_get_timeline_name,
224 .enable_signaling = dma_fence_chain_enable_signaling,
225 .signaled = dma_fence_chain_signaled,
226 .release = dma_fence_chain_release,
227 .set_deadline = dma_fence_chain_set_deadline,
228};
229EXPORT_SYMBOL(dma_fence_chain_ops);
230
231/**
232 * dma_fence_chain_init - initialize a fence chain
233 * @chain: the chain node to initialize
234 * @prev: the previous fence
235 * @fence: the current fence
236 * @seqno: the sequence number to use for the fence chain
237 *
238 * Initialize a new chain node and either start a new chain or add the node to
239 * the existing chain of the previous fence.
240 */
241void dma_fence_chain_init(struct dma_fence_chain *chain,
242 struct dma_fence *prev,
243 struct dma_fence *fence,
244 uint64_t seqno)
245{
246 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
247 uint64_t context;
248
249 spin_lock_init(&chain->lock);
250 rcu_assign_pointer(chain->prev, prev);
251 chain->fence = fence;
252 chain->prev_seqno = 0;
253
254 /* Try to reuse the context of the previous chain node. */
255 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
256 context = prev->context;
257 chain->prev_seqno = prev->seqno;
258 } else {
259 context = dma_fence_context_alloc(1);
260 /* Make sure that we always have a valid sequence number. */
261 if (prev_chain)
262 seqno = max(prev->seqno, seqno);
263 }
264
265 dma_fence_init(&chain->base, &dma_fence_chain_ops,
266 &chain->lock, context, seqno);
267
268 /*
269 * Chaining dma_fence_chain container together is only allowed through
270 * the prev fence and not through the contained fence.
271 *
272 * The correct way of handling this is to flatten out the fence
273 * structure into a dma_fence_array by the caller instead.
274 */
275 WARN_ON(dma_fence_is_chain(fence));
276}
277EXPORT_SYMBOL(dma_fence_chain_init);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * fence-chain: chain fences together in a timeline
4 *
5 * Copyright (C) 2018 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
8 */
9
10#include <linux/dma-fence-chain.h>
11
12static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
13
14/**
15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
16 * @chain: chain node to get the previous node from
17 *
18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
19 * chain node.
20 */
21static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
22{
23 struct dma_fence *prev;
24
25 rcu_read_lock();
26 prev = dma_fence_get_rcu_safe(&chain->prev);
27 rcu_read_unlock();
28 return prev;
29}
30
31/**
32 * dma_fence_chain_walk - chain walking function
33 * @fence: current chain node
34 *
35 * Walk the chain to the next node. Returns the next fence or NULL if we are at
36 * the end of the chain. Garbage collects chain nodes which are already
37 * signaled.
38 */
39struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
40{
41 struct dma_fence_chain *chain, *prev_chain;
42 struct dma_fence *prev, *replacement, *tmp;
43
44 chain = to_dma_fence_chain(fence);
45 if (!chain) {
46 dma_fence_put(fence);
47 return NULL;
48 }
49
50 while ((prev = dma_fence_chain_get_prev(chain))) {
51
52 prev_chain = to_dma_fence_chain(prev);
53 if (prev_chain) {
54 if (!dma_fence_is_signaled(prev_chain->fence))
55 break;
56
57 replacement = dma_fence_chain_get_prev(prev_chain);
58 } else {
59 if (!dma_fence_is_signaled(prev))
60 break;
61
62 replacement = NULL;
63 }
64
65 tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement);
66 if (tmp == prev)
67 dma_fence_put(tmp);
68 else
69 dma_fence_put(replacement);
70 dma_fence_put(prev);
71 }
72
73 dma_fence_put(fence);
74 return prev;
75}
76EXPORT_SYMBOL(dma_fence_chain_walk);
77
78/**
79 * dma_fence_chain_find_seqno - find fence chain node by seqno
80 * @pfence: pointer to the chain node where to start
81 * @seqno: the sequence number to search for
82 *
83 * Advance the fence pointer to the chain node which will signal this sequence
84 * number. If no sequence number is provided then this is a no-op.
85 *
86 * Returns EINVAL if the fence is not a chain node or the sequence number has
87 * not yet advanced far enough.
88 */
89int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
90{
91 struct dma_fence_chain *chain;
92
93 if (!seqno)
94 return 0;
95
96 chain = to_dma_fence_chain(*pfence);
97 if (!chain || chain->base.seqno < seqno)
98 return -EINVAL;
99
100 dma_fence_chain_for_each(*pfence, &chain->base) {
101 if ((*pfence)->context != chain->base.context ||
102 to_dma_fence_chain(*pfence)->prev_seqno < seqno)
103 break;
104 }
105 dma_fence_put(&chain->base);
106
107 return 0;
108}
109EXPORT_SYMBOL(dma_fence_chain_find_seqno);
110
111static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
112{
113 return "dma_fence_chain";
114}
115
116static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
117{
118 return "unbound";
119}
120
121static void dma_fence_chain_irq_work(struct irq_work *work)
122{
123 struct dma_fence_chain *chain;
124
125 chain = container_of(work, typeof(*chain), work);
126
127 /* Try to rearm the callback */
128 if (!dma_fence_chain_enable_signaling(&chain->base))
129 /* Ok, we are done. No more unsignaled fences left */
130 dma_fence_signal(&chain->base);
131 dma_fence_put(&chain->base);
132}
133
134static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
135{
136 struct dma_fence_chain *chain;
137
138 chain = container_of(cb, typeof(*chain), cb);
139 irq_work_queue(&chain->work);
140 dma_fence_put(f);
141}
142
143static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
144{
145 struct dma_fence_chain *head = to_dma_fence_chain(fence);
146
147 dma_fence_get(&head->base);
148 dma_fence_chain_for_each(fence, &head->base) {
149 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
150 struct dma_fence *f = chain ? chain->fence : fence;
151
152 dma_fence_get(f);
153 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
154 dma_fence_put(fence);
155 return true;
156 }
157 dma_fence_put(f);
158 }
159 dma_fence_put(&head->base);
160 return false;
161}
162
163static bool dma_fence_chain_signaled(struct dma_fence *fence)
164{
165 dma_fence_chain_for_each(fence, fence) {
166 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
167 struct dma_fence *f = chain ? chain->fence : fence;
168
169 if (!dma_fence_is_signaled(f)) {
170 dma_fence_put(fence);
171 return false;
172 }
173 }
174
175 return true;
176}
177
178static void dma_fence_chain_release(struct dma_fence *fence)
179{
180 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
181 struct dma_fence *prev;
182
183 /* Manually unlink the chain as much as possible to avoid recursion
184 * and potential stack overflow.
185 */
186 while ((prev = rcu_dereference_protected(chain->prev, true))) {
187 struct dma_fence_chain *prev_chain;
188
189 if (kref_read(&prev->refcount) > 1)
190 break;
191
192 prev_chain = to_dma_fence_chain(prev);
193 if (!prev_chain)
194 break;
195
196 /* No need for atomic operations since we hold the last
197 * reference to prev_chain.
198 */
199 chain->prev = prev_chain->prev;
200 RCU_INIT_POINTER(prev_chain->prev, NULL);
201 dma_fence_put(prev);
202 }
203 dma_fence_put(prev);
204
205 dma_fence_put(chain->fence);
206 dma_fence_free(fence);
207}
208
209const struct dma_fence_ops dma_fence_chain_ops = {
210 .use_64bit_seqno = true,
211 .get_driver_name = dma_fence_chain_get_driver_name,
212 .get_timeline_name = dma_fence_chain_get_timeline_name,
213 .enable_signaling = dma_fence_chain_enable_signaling,
214 .signaled = dma_fence_chain_signaled,
215 .release = dma_fence_chain_release,
216};
217EXPORT_SYMBOL(dma_fence_chain_ops);
218
219/**
220 * dma_fence_chain_init - initialize a fence chain
221 * @chain: the chain node to initialize
222 * @prev: the previous fence
223 * @fence: the current fence
224 *
225 * Initialize a new chain node and either start a new chain or add the node to
226 * the existing chain of the previous fence.
227 */
228void dma_fence_chain_init(struct dma_fence_chain *chain,
229 struct dma_fence *prev,
230 struct dma_fence *fence,
231 uint64_t seqno)
232{
233 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
234 uint64_t context;
235
236 spin_lock_init(&chain->lock);
237 rcu_assign_pointer(chain->prev, prev);
238 chain->fence = fence;
239 chain->prev_seqno = 0;
240 init_irq_work(&chain->work, dma_fence_chain_irq_work);
241
242 /* Try to reuse the context of the previous chain node. */
243 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
244 context = prev->context;
245 chain->prev_seqno = prev->seqno;
246 } else {
247 context = dma_fence_context_alloc(1);
248 /* Make sure that we always have a valid sequence number. */
249 if (prev_chain)
250 seqno = max(prev->seqno, seqno);
251 }
252
253 dma_fence_init(&chain->base, &dma_fence_chain_ops,
254 &chain->lock, context, seqno);
255}
256EXPORT_SYMBOL(dma_fence_chain_init);