Loading...
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2016-2018 Intel Corporation
5 */
6
7#include "gt/intel_gt_types.h"
8
9#include "i915_drv.h"
10
11#include "i915_active.h"
12#include "i915_syncmap.h"
13#include "gt/intel_timeline.h"
14
15#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
16#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
17
18struct intel_timeline_hwsp {
19 struct intel_gt *gt;
20 struct intel_gt_timelines *gt_timelines;
21 struct list_head free_link;
22 struct i915_vma *vma;
23 u64 free_bitmap;
24};
25
26struct intel_timeline_cacheline {
27 struct i915_active active;
28 struct intel_timeline_hwsp *hwsp;
29 void *vaddr;
30#define CACHELINE_BITS 6
31#define CACHELINE_FREE CACHELINE_BITS
32};
33
34static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
35{
36 struct drm_i915_private *i915 = gt->i915;
37 struct drm_i915_gem_object *obj;
38 struct i915_vma *vma;
39
40 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
41 if (IS_ERR(obj))
42 return ERR_CAST(obj);
43
44 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
45
46 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
47 if (IS_ERR(vma))
48 i915_gem_object_put(obj);
49
50 return vma;
51}
52
53static struct i915_vma *
54hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
55{
56 struct intel_gt_timelines *gt = &timeline->gt->timelines;
57 struct intel_timeline_hwsp *hwsp;
58
59 BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
60
61 spin_lock_irq(>->hwsp_lock);
62
63 /* hwsp_free_list only contains HWSP that have available cachelines */
64 hwsp = list_first_entry_or_null(>->hwsp_free_list,
65 typeof(*hwsp), free_link);
66 if (!hwsp) {
67 struct i915_vma *vma;
68
69 spin_unlock_irq(>->hwsp_lock);
70
71 hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
72 if (!hwsp)
73 return ERR_PTR(-ENOMEM);
74
75 vma = __hwsp_alloc(timeline->gt);
76 if (IS_ERR(vma)) {
77 kfree(hwsp);
78 return vma;
79 }
80
81 vma->private = hwsp;
82 hwsp->gt = timeline->gt;
83 hwsp->vma = vma;
84 hwsp->free_bitmap = ~0ull;
85 hwsp->gt_timelines = gt;
86
87 spin_lock_irq(>->hwsp_lock);
88 list_add(&hwsp->free_link, >->hwsp_free_list);
89 }
90
91 GEM_BUG_ON(!hwsp->free_bitmap);
92 *cacheline = __ffs64(hwsp->free_bitmap);
93 hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
94 if (!hwsp->free_bitmap)
95 list_del(&hwsp->free_link);
96
97 spin_unlock_irq(>->hwsp_lock);
98
99 GEM_BUG_ON(hwsp->vma->private != hwsp);
100 return hwsp->vma;
101}
102
103static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
104{
105 struct intel_gt_timelines *gt = hwsp->gt_timelines;
106 unsigned long flags;
107
108 spin_lock_irqsave(>->hwsp_lock, flags);
109
110 /* As a cacheline becomes available, publish the HWSP on the freelist */
111 if (!hwsp->free_bitmap)
112 list_add_tail(&hwsp->free_link, >->hwsp_free_list);
113
114 GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
115 hwsp->free_bitmap |= BIT_ULL(cacheline);
116
117 /* And if no one is left using it, give the page back to the system */
118 if (hwsp->free_bitmap == ~0ull) {
119 i915_vma_put(hwsp->vma);
120 list_del(&hwsp->free_link);
121 kfree(hwsp);
122 }
123
124 spin_unlock_irqrestore(>->hwsp_lock, flags);
125}
126
127static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
128{
129 GEM_BUG_ON(!i915_active_is_idle(&cl->active));
130
131 i915_gem_object_unpin_map(cl->hwsp->vma->obj);
132 i915_vma_put(cl->hwsp->vma);
133 __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
134
135 i915_active_fini(&cl->active);
136 kfree(cl);
137}
138
139static void __cacheline_retire(struct i915_active *active)
140{
141 struct intel_timeline_cacheline *cl =
142 container_of(active, typeof(*cl), active);
143
144 i915_vma_unpin(cl->hwsp->vma);
145 if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
146 __idle_cacheline_free(cl);
147}
148
149static int __cacheline_active(struct i915_active *active)
150{
151 struct intel_timeline_cacheline *cl =
152 container_of(active, typeof(*cl), active);
153
154 __i915_vma_pin(cl->hwsp->vma);
155 return 0;
156}
157
158static struct intel_timeline_cacheline *
159cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
160{
161 struct intel_timeline_cacheline *cl;
162 void *vaddr;
163
164 GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
165
166 cl = kmalloc(sizeof(*cl), GFP_KERNEL);
167 if (!cl)
168 return ERR_PTR(-ENOMEM);
169
170 vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
171 if (IS_ERR(vaddr)) {
172 kfree(cl);
173 return ERR_CAST(vaddr);
174 }
175
176 i915_vma_get(hwsp->vma);
177 cl->hwsp = hwsp;
178 cl->vaddr = page_pack_bits(vaddr, cacheline);
179
180 i915_active_init(hwsp->gt->i915, &cl->active,
181 __cacheline_active, __cacheline_retire);
182
183 return cl;
184}
185
186static void cacheline_acquire(struct intel_timeline_cacheline *cl)
187{
188 if (cl)
189 i915_active_acquire(&cl->active);
190}
191
192static void cacheline_release(struct intel_timeline_cacheline *cl)
193{
194 if (cl)
195 i915_active_release(&cl->active);
196}
197
198static void cacheline_free(struct intel_timeline_cacheline *cl)
199{
200 GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
201 cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
202
203 if (i915_active_is_idle(&cl->active))
204 __idle_cacheline_free(cl);
205}
206
207int intel_timeline_init(struct intel_timeline *timeline,
208 struct intel_gt *gt,
209 struct i915_vma *hwsp)
210{
211 void *vaddr;
212
213 kref_init(&timeline->kref);
214 atomic_set(&timeline->pin_count, 0);
215
216 timeline->gt = gt;
217
218 timeline->has_initial_breadcrumb = !hwsp;
219 timeline->hwsp_cacheline = NULL;
220
221 if (!hwsp) {
222 struct intel_timeline_cacheline *cl;
223 unsigned int cacheline;
224
225 hwsp = hwsp_alloc(timeline, &cacheline);
226 if (IS_ERR(hwsp))
227 return PTR_ERR(hwsp);
228
229 cl = cacheline_alloc(hwsp->private, cacheline);
230 if (IS_ERR(cl)) {
231 __idle_hwsp_free(hwsp->private, cacheline);
232 return PTR_ERR(cl);
233 }
234
235 timeline->hwsp_cacheline = cl;
236 timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
237
238 vaddr = page_mask_bits(cl->vaddr);
239 } else {
240 timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
241
242 vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
243 if (IS_ERR(vaddr))
244 return PTR_ERR(vaddr);
245 }
246
247 timeline->hwsp_seqno =
248 memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
249
250 timeline->hwsp_ggtt = i915_vma_get(hwsp);
251 GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
252
253 timeline->fence_context = dma_fence_context_alloc(1);
254
255 mutex_init(&timeline->mutex);
256
257 INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex);
258 INIT_LIST_HEAD(&timeline->requests);
259
260 i915_syncmap_init(&timeline->sync);
261
262 return 0;
263}
264
265static void timelines_init(struct intel_gt *gt)
266{
267 struct intel_gt_timelines *timelines = >->timelines;
268
269 spin_lock_init(&timelines->lock);
270 INIT_LIST_HEAD(&timelines->active_list);
271
272 spin_lock_init(&timelines->hwsp_lock);
273 INIT_LIST_HEAD(&timelines->hwsp_free_list);
274}
275
276void intel_timelines_init(struct drm_i915_private *i915)
277{
278 timelines_init(&i915->gt);
279}
280
281void intel_timeline_fini(struct intel_timeline *timeline)
282{
283 GEM_BUG_ON(atomic_read(&timeline->pin_count));
284 GEM_BUG_ON(!list_empty(&timeline->requests));
285
286 if (timeline->hwsp_cacheline)
287 cacheline_free(timeline->hwsp_cacheline);
288 else
289 i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
290
291 i915_vma_put(timeline->hwsp_ggtt);
292}
293
294struct intel_timeline *
295intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp)
296{
297 struct intel_timeline *timeline;
298 int err;
299
300 timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
301 if (!timeline)
302 return ERR_PTR(-ENOMEM);
303
304 err = intel_timeline_init(timeline, gt, global_hwsp);
305 if (err) {
306 kfree(timeline);
307 return ERR_PTR(err);
308 }
309
310 return timeline;
311}
312
313int intel_timeline_pin(struct intel_timeline *tl)
314{
315 int err;
316
317 if (atomic_add_unless(&tl->pin_count, 1, 0))
318 return 0;
319
320 err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH);
321 if (err)
322 return err;
323
324 tl->hwsp_offset =
325 i915_ggtt_offset(tl->hwsp_ggtt) +
326 offset_in_page(tl->hwsp_offset);
327
328 cacheline_acquire(tl->hwsp_cacheline);
329 if (atomic_fetch_inc(&tl->pin_count)) {
330 cacheline_release(tl->hwsp_cacheline);
331 __i915_vma_unpin(tl->hwsp_ggtt);
332 }
333
334 return 0;
335}
336
337void intel_timeline_enter(struct intel_timeline *tl)
338{
339 struct intel_gt_timelines *timelines = &tl->gt->timelines;
340 unsigned long flags;
341
342 lockdep_assert_held(&tl->mutex);
343
344 GEM_BUG_ON(!atomic_read(&tl->pin_count));
345 if (tl->active_count++)
346 return;
347 GEM_BUG_ON(!tl->active_count); /* overflow? */
348
349 spin_lock_irqsave(&timelines->lock, flags);
350 list_add(&tl->link, &timelines->active_list);
351 spin_unlock_irqrestore(&timelines->lock, flags);
352}
353
354void intel_timeline_exit(struct intel_timeline *tl)
355{
356 struct intel_gt_timelines *timelines = &tl->gt->timelines;
357 unsigned long flags;
358
359 lockdep_assert_held(&tl->mutex);
360
361 GEM_BUG_ON(!tl->active_count);
362 if (--tl->active_count)
363 return;
364
365 spin_lock_irqsave(&timelines->lock, flags);
366 list_del(&tl->link);
367 spin_unlock_irqrestore(&timelines->lock, flags);
368
369 /*
370 * Since this timeline is idle, all bariers upon which we were waiting
371 * must also be complete and so we can discard the last used barriers
372 * without loss of information.
373 */
374 i915_syncmap_free(&tl->sync);
375}
376
377static u32 timeline_advance(struct intel_timeline *tl)
378{
379 GEM_BUG_ON(!atomic_read(&tl->pin_count));
380 GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
381
382 return tl->seqno += 1 + tl->has_initial_breadcrumb;
383}
384
385static void timeline_rollback(struct intel_timeline *tl)
386{
387 tl->seqno -= 1 + tl->has_initial_breadcrumb;
388}
389
390static noinline int
391__intel_timeline_get_seqno(struct intel_timeline *tl,
392 struct i915_request *rq,
393 u32 *seqno)
394{
395 struct intel_timeline_cacheline *cl;
396 unsigned int cacheline;
397 struct i915_vma *vma;
398 void *vaddr;
399 int err;
400
401 /*
402 * If there is an outstanding GPU reference to this cacheline,
403 * such as it being sampled by a HW semaphore on another timeline,
404 * we cannot wraparound our seqno value (the HW semaphore does
405 * a strict greater-than-or-equals compare, not i915_seqno_passed).
406 * So if the cacheline is still busy, we must detach ourselves
407 * from it and leave it inflight alongside its users.
408 *
409 * However, if nobody is watching and we can guarantee that nobody
410 * will, we could simply reuse the same cacheline.
411 *
412 * if (i915_active_request_is_signaled(&tl->last_request) &&
413 * i915_active_is_signaled(&tl->hwsp_cacheline->active))
414 * return 0;
415 *
416 * That seems unlikely for a busy timeline that needed to wrap in
417 * the first place, so just replace the cacheline.
418 */
419
420 vma = hwsp_alloc(tl, &cacheline);
421 if (IS_ERR(vma)) {
422 err = PTR_ERR(vma);
423 goto err_rollback;
424 }
425
426 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
427 if (err) {
428 __idle_hwsp_free(vma->private, cacheline);
429 goto err_rollback;
430 }
431
432 cl = cacheline_alloc(vma->private, cacheline);
433 if (IS_ERR(cl)) {
434 err = PTR_ERR(cl);
435 __idle_hwsp_free(vma->private, cacheline);
436 goto err_unpin;
437 }
438 GEM_BUG_ON(cl->hwsp->vma != vma);
439
440 /*
441 * Attach the old cacheline to the current request, so that we only
442 * free it after the current request is retired, which ensures that
443 * all writes into the cacheline from previous requests are complete.
444 */
445 err = i915_active_ref(&tl->hwsp_cacheline->active, tl, rq);
446 if (err)
447 goto err_cacheline;
448
449 cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
450 cacheline_free(tl->hwsp_cacheline);
451
452 i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
453 i915_vma_put(tl->hwsp_ggtt);
454
455 tl->hwsp_ggtt = i915_vma_get(vma);
456
457 vaddr = page_mask_bits(cl->vaddr);
458 tl->hwsp_offset = cacheline * CACHELINE_BYTES;
459 tl->hwsp_seqno =
460 memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
461
462 tl->hwsp_offset += i915_ggtt_offset(vma);
463
464 cacheline_acquire(cl);
465 tl->hwsp_cacheline = cl;
466
467 *seqno = timeline_advance(tl);
468 GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
469 return 0;
470
471err_cacheline:
472 cacheline_free(cl);
473err_unpin:
474 i915_vma_unpin(vma);
475err_rollback:
476 timeline_rollback(tl);
477 return err;
478}
479
480int intel_timeline_get_seqno(struct intel_timeline *tl,
481 struct i915_request *rq,
482 u32 *seqno)
483{
484 *seqno = timeline_advance(tl);
485
486 /* Replace the HWSP on wraparound for HW semaphores */
487 if (unlikely(!*seqno && tl->hwsp_cacheline))
488 return __intel_timeline_get_seqno(tl, rq, seqno);
489
490 return 0;
491}
492
493static int cacheline_ref(struct intel_timeline_cacheline *cl,
494 struct i915_request *rq)
495{
496 return i915_active_ref(&cl->active, rq->timeline, rq);
497}
498
499int intel_timeline_read_hwsp(struct i915_request *from,
500 struct i915_request *to,
501 u32 *hwsp)
502{
503 struct intel_timeline_cacheline *cl = from->hwsp_cacheline;
504 struct intel_timeline *tl = from->timeline;
505 int err;
506
507 GEM_BUG_ON(to->timeline == tl);
508
509 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);
510 err = i915_request_completed(from);
511 if (!err)
512 err = cacheline_ref(cl, to);
513 if (!err) {
514 if (likely(cl == tl->hwsp_cacheline)) {
515 *hwsp = tl->hwsp_offset;
516 } else { /* across a seqno wrap, recover the original offset */
517 *hwsp = i915_ggtt_offset(cl->hwsp->vma) +
518 ptr_unmask_bits(cl->vaddr, CACHELINE_BITS) *
519 CACHELINE_BYTES;
520 }
521 }
522 mutex_unlock(&tl->mutex);
523
524 return err;
525}
526
527void intel_timeline_unpin(struct intel_timeline *tl)
528{
529 GEM_BUG_ON(!atomic_read(&tl->pin_count));
530 if (!atomic_dec_and_test(&tl->pin_count))
531 return;
532
533 cacheline_release(tl->hwsp_cacheline);
534
535 __i915_vma_unpin(tl->hwsp_ggtt);
536}
537
538void __intel_timeline_free(struct kref *kref)
539{
540 struct intel_timeline *timeline =
541 container_of(kref, typeof(*timeline), kref);
542
543 intel_timeline_fini(timeline);
544 kfree(timeline);
545}
546
547static void timelines_fini(struct intel_gt *gt)
548{
549 struct intel_gt_timelines *timelines = >->timelines;
550
551 GEM_BUG_ON(!list_empty(&timelines->active_list));
552 GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
553}
554
555void intel_timelines_fini(struct drm_i915_private *i915)
556{
557 timelines_fini(&i915->gt);
558}
559
560#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
561#include "gt/selftests/mock_timeline.c"
562#include "gt/selftest_timeline.c"
563#endif
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2016-2018 Intel Corporation
4 */
5
6#include <drm/drm_cache.h>
7
8#include "gem/i915_gem_internal.h"
9
10#include "i915_active.h"
11#include "i915_drv.h"
12#include "i915_syncmap.h"
13#include "intel_gt.h"
14#include "intel_ring.h"
15#include "intel_timeline.h"
16
17#define TIMELINE_SEQNO_BYTES 8
18
19static struct i915_vma *hwsp_alloc(struct intel_gt *gt)
20{
21 struct drm_i915_private *i915 = gt->i915;
22 struct drm_i915_gem_object *obj;
23 struct i915_vma *vma;
24
25 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
26 if (IS_ERR(obj))
27 return ERR_CAST(obj);
28
29 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
30
31 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
32 if (IS_ERR(vma))
33 i915_gem_object_put(obj);
34
35 return vma;
36}
37
38static void __timeline_retire(struct i915_active *active)
39{
40 struct intel_timeline *tl =
41 container_of(active, typeof(*tl), active);
42
43 i915_vma_unpin(tl->hwsp_ggtt);
44 intel_timeline_put(tl);
45}
46
47static int __timeline_active(struct i915_active *active)
48{
49 struct intel_timeline *tl =
50 container_of(active, typeof(*tl), active);
51
52 __i915_vma_pin(tl->hwsp_ggtt);
53 intel_timeline_get(tl);
54 return 0;
55}
56
57I915_SELFTEST_EXPORT int
58intel_timeline_pin_map(struct intel_timeline *timeline)
59{
60 struct drm_i915_gem_object *obj = timeline->hwsp_ggtt->obj;
61 u32 ofs = offset_in_page(timeline->hwsp_offset);
62 void *vaddr;
63
64 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
65 if (IS_ERR(vaddr))
66 return PTR_ERR(vaddr);
67
68 timeline->hwsp_map = vaddr;
69 timeline->hwsp_seqno = memset(vaddr + ofs, 0, TIMELINE_SEQNO_BYTES);
70 drm_clflush_virt_range(vaddr + ofs, TIMELINE_SEQNO_BYTES);
71
72 return 0;
73}
74
75static int intel_timeline_init(struct intel_timeline *timeline,
76 struct intel_gt *gt,
77 struct i915_vma *hwsp,
78 unsigned int offset)
79{
80 kref_init(&timeline->kref);
81 atomic_set(&timeline->pin_count, 0);
82
83 timeline->gt = gt;
84
85 if (hwsp) {
86 timeline->hwsp_offset = offset;
87 timeline->hwsp_ggtt = i915_vma_get(hwsp);
88 } else {
89 timeline->has_initial_breadcrumb = true;
90 hwsp = hwsp_alloc(gt);
91 if (IS_ERR(hwsp))
92 return PTR_ERR(hwsp);
93 timeline->hwsp_ggtt = hwsp;
94 }
95
96 timeline->hwsp_map = NULL;
97 timeline->hwsp_seqno = (void *)(long)timeline->hwsp_offset;
98
99 GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
100
101 timeline->fence_context = dma_fence_context_alloc(1);
102
103 mutex_init(&timeline->mutex);
104
105 INIT_ACTIVE_FENCE(&timeline->last_request);
106 INIT_LIST_HEAD(&timeline->requests);
107
108 i915_syncmap_init(&timeline->sync);
109 i915_active_init(&timeline->active, __timeline_active,
110 __timeline_retire, 0);
111
112 return 0;
113}
114
115void intel_gt_init_timelines(struct intel_gt *gt)
116{
117 struct intel_gt_timelines *timelines = >->timelines;
118
119 spin_lock_init(&timelines->lock);
120 INIT_LIST_HEAD(&timelines->active_list);
121}
122
123static void intel_timeline_fini(struct rcu_head *rcu)
124{
125 struct intel_timeline *timeline =
126 container_of(rcu, struct intel_timeline, rcu);
127
128 if (timeline->hwsp_map)
129 i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
130
131 i915_vma_put(timeline->hwsp_ggtt);
132 i915_active_fini(&timeline->active);
133
134 /*
135 * A small race exists between intel_gt_retire_requests_timeout and
136 * intel_timeline_exit which could result in the syncmap not getting
137 * free'd. Rather than work to hard to seal this race, simply cleanup
138 * the syncmap on fini.
139 */
140 i915_syncmap_free(&timeline->sync);
141
142 kfree(timeline);
143}
144
145struct intel_timeline *
146__intel_timeline_create(struct intel_gt *gt,
147 struct i915_vma *global_hwsp,
148 unsigned int offset)
149{
150 struct intel_timeline *timeline;
151 int err;
152
153 timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
154 if (!timeline)
155 return ERR_PTR(-ENOMEM);
156
157 err = intel_timeline_init(timeline, gt, global_hwsp, offset);
158 if (err) {
159 kfree(timeline);
160 return ERR_PTR(err);
161 }
162
163 return timeline;
164}
165
166struct intel_timeline *
167intel_timeline_create_from_engine(struct intel_engine_cs *engine,
168 unsigned int offset)
169{
170 struct i915_vma *hwsp = engine->status_page.vma;
171 struct intel_timeline *tl;
172
173 tl = __intel_timeline_create(engine->gt, hwsp, offset);
174 if (IS_ERR(tl))
175 return tl;
176
177 /* Borrow a nearby lock; we only create these timelines during init */
178 mutex_lock(&hwsp->vm->mutex);
179 list_add_tail(&tl->engine_link, &engine->status_page.timelines);
180 mutex_unlock(&hwsp->vm->mutex);
181
182 return tl;
183}
184
185void __intel_timeline_pin(struct intel_timeline *tl)
186{
187 GEM_BUG_ON(!atomic_read(&tl->pin_count));
188 atomic_inc(&tl->pin_count);
189}
190
191int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww)
192{
193 int err;
194
195 if (atomic_add_unless(&tl->pin_count, 1, 0))
196 return 0;
197
198 if (!tl->hwsp_map) {
199 err = intel_timeline_pin_map(tl);
200 if (err)
201 return err;
202 }
203
204 err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
205 if (err)
206 return err;
207
208 tl->hwsp_offset =
209 i915_ggtt_offset(tl->hwsp_ggtt) +
210 offset_in_page(tl->hwsp_offset);
211 GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
212 tl->fence_context, tl->hwsp_offset);
213
214 i915_active_acquire(&tl->active);
215 if (atomic_fetch_inc(&tl->pin_count)) {
216 i915_active_release(&tl->active);
217 __i915_vma_unpin(tl->hwsp_ggtt);
218 }
219
220 return 0;
221}
222
223void intel_timeline_reset_seqno(const struct intel_timeline *tl)
224{
225 u32 *hwsp_seqno = (u32 *)tl->hwsp_seqno;
226 /* Must be pinned to be writable, and no requests in flight. */
227 GEM_BUG_ON(!atomic_read(&tl->pin_count));
228
229 memset(hwsp_seqno + 1, 0, TIMELINE_SEQNO_BYTES - sizeof(*hwsp_seqno));
230 WRITE_ONCE(*hwsp_seqno, tl->seqno);
231 drm_clflush_virt_range(hwsp_seqno, TIMELINE_SEQNO_BYTES);
232}
233
234void intel_timeline_enter(struct intel_timeline *tl)
235{
236 struct intel_gt_timelines *timelines = &tl->gt->timelines;
237
238 /*
239 * Pretend we are serialised by the timeline->mutex.
240 *
241 * While generally true, there are a few exceptions to the rule
242 * for the engine->kernel_context being used to manage power
243 * transitions. As the engine_park may be called from under any
244 * timeline, it uses the power mutex as a global serialisation
245 * lock to prevent any other request entering its timeline.
246 *
247 * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
248 *
249 * However, intel_gt_retire_request() does not know which engine
250 * it is retiring along and so cannot partake in the engine-pm
251 * barrier, and there we use the tl->active_count as a means to
252 * pin the timeline in the active_list while the locks are dropped.
253 * Ergo, as that is outside of the engine-pm barrier, we need to
254 * use atomic to manipulate tl->active_count.
255 */
256 lockdep_assert_held(&tl->mutex);
257
258 if (atomic_add_unless(&tl->active_count, 1, 0))
259 return;
260
261 spin_lock(&timelines->lock);
262 if (!atomic_fetch_inc(&tl->active_count)) {
263 /*
264 * The HWSP is volatile, and may have been lost while inactive,
265 * e.g. across suspend/resume. Be paranoid, and ensure that
266 * the HWSP value matches our seqno so we don't proclaim
267 * the next request as already complete.
268 */
269 intel_timeline_reset_seqno(tl);
270 list_add_tail(&tl->link, &timelines->active_list);
271 }
272 spin_unlock(&timelines->lock);
273}
274
275void intel_timeline_exit(struct intel_timeline *tl)
276{
277 struct intel_gt_timelines *timelines = &tl->gt->timelines;
278
279 /* See intel_timeline_enter() */
280 lockdep_assert_held(&tl->mutex);
281
282 GEM_BUG_ON(!atomic_read(&tl->active_count));
283 if (atomic_add_unless(&tl->active_count, -1, 1))
284 return;
285
286 spin_lock(&timelines->lock);
287 if (atomic_dec_and_test(&tl->active_count))
288 list_del(&tl->link);
289 spin_unlock(&timelines->lock);
290
291 /*
292 * Since this timeline is idle, all bariers upon which we were waiting
293 * must also be complete and so we can discard the last used barriers
294 * without loss of information.
295 */
296 i915_syncmap_free(&tl->sync);
297}
298
299static u32 timeline_advance(struct intel_timeline *tl)
300{
301 GEM_BUG_ON(!atomic_read(&tl->pin_count));
302 GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
303
304 return tl->seqno += 1 + tl->has_initial_breadcrumb;
305}
306
307static noinline int
308__intel_timeline_get_seqno(struct intel_timeline *tl,
309 u32 *seqno)
310{
311 u32 next_ofs = offset_in_page(tl->hwsp_offset + TIMELINE_SEQNO_BYTES);
312
313 /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
314 if (TIMELINE_SEQNO_BYTES <= BIT(5) && (next_ofs & BIT(5)))
315 next_ofs = offset_in_page(next_ofs + BIT(5));
316
317 tl->hwsp_offset = i915_ggtt_offset(tl->hwsp_ggtt) + next_ofs;
318 tl->hwsp_seqno = tl->hwsp_map + next_ofs;
319 intel_timeline_reset_seqno(tl);
320
321 *seqno = timeline_advance(tl);
322 GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
323 return 0;
324}
325
326int intel_timeline_get_seqno(struct intel_timeline *tl,
327 struct i915_request *rq,
328 u32 *seqno)
329{
330 *seqno = timeline_advance(tl);
331
332 /* Replace the HWSP on wraparound for HW semaphores */
333 if (unlikely(!*seqno && tl->has_initial_breadcrumb))
334 return __intel_timeline_get_seqno(tl, seqno);
335
336 return 0;
337}
338
339int intel_timeline_read_hwsp(struct i915_request *from,
340 struct i915_request *to,
341 u32 *hwsp)
342{
343 struct intel_timeline *tl;
344 int err;
345
346 rcu_read_lock();
347 tl = rcu_dereference(from->timeline);
348 if (i915_request_signaled(from) ||
349 !i915_active_acquire_if_busy(&tl->active))
350 tl = NULL;
351
352 if (tl) {
353 /* hwsp_offset may wraparound, so use from->hwsp_seqno */
354 *hwsp = i915_ggtt_offset(tl->hwsp_ggtt) +
355 offset_in_page(from->hwsp_seqno);
356 }
357
358 /* ensure we wait on the right request, if not, we completed */
359 if (tl && __i915_request_is_complete(from)) {
360 i915_active_release(&tl->active);
361 tl = NULL;
362 }
363 rcu_read_unlock();
364
365 if (!tl)
366 return 1;
367
368 /* Can't do semaphore waits on kernel context */
369 if (!tl->has_initial_breadcrumb) {
370 err = -EINVAL;
371 goto out;
372 }
373
374 err = i915_active_add_request(&tl->active, to);
375
376out:
377 i915_active_release(&tl->active);
378 return err;
379}
380
381void intel_timeline_unpin(struct intel_timeline *tl)
382{
383 GEM_BUG_ON(!atomic_read(&tl->pin_count));
384 if (!atomic_dec_and_test(&tl->pin_count))
385 return;
386
387 i915_active_release(&tl->active);
388 __i915_vma_unpin(tl->hwsp_ggtt);
389}
390
391void __intel_timeline_free(struct kref *kref)
392{
393 struct intel_timeline *timeline =
394 container_of(kref, typeof(*timeline), kref);
395
396 GEM_BUG_ON(atomic_read(&timeline->pin_count));
397 GEM_BUG_ON(!list_empty(&timeline->requests));
398 GEM_BUG_ON(timeline->retire);
399
400 call_rcu(&timeline->rcu, intel_timeline_fini);
401}
402
403void intel_gt_fini_timelines(struct intel_gt *gt)
404{
405 struct intel_gt_timelines *timelines = >->timelines;
406
407 GEM_BUG_ON(!list_empty(&timelines->active_list));
408}
409
410void intel_gt_show_timelines(struct intel_gt *gt,
411 struct drm_printer *m,
412 void (*show_request)(struct drm_printer *m,
413 const struct i915_request *rq,
414 const char *prefix,
415 int indent))
416{
417 struct intel_gt_timelines *timelines = >->timelines;
418 struct intel_timeline *tl, *tn;
419 LIST_HEAD(free);
420
421 spin_lock(&timelines->lock);
422 list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
423 unsigned long count, ready, inflight;
424 struct i915_request *rq, *rn;
425 struct dma_fence *fence;
426
427 if (!mutex_trylock(&tl->mutex)) {
428 drm_printf(m, "Timeline %llx: busy; skipping\n",
429 tl->fence_context);
430 continue;
431 }
432
433 intel_timeline_get(tl);
434 GEM_BUG_ON(!atomic_read(&tl->active_count));
435 atomic_inc(&tl->active_count); /* pin the list element */
436 spin_unlock(&timelines->lock);
437
438 count = 0;
439 ready = 0;
440 inflight = 0;
441 list_for_each_entry_safe(rq, rn, &tl->requests, link) {
442 if (i915_request_completed(rq))
443 continue;
444
445 count++;
446 if (i915_request_is_ready(rq))
447 ready++;
448 if (i915_request_is_active(rq))
449 inflight++;
450 }
451
452 drm_printf(m, "Timeline %llx: { ", tl->fence_context);
453 drm_printf(m, "count: %lu, ready: %lu, inflight: %lu",
454 count, ready, inflight);
455 drm_printf(m, ", seqno: { current: %d, last: %d }",
456 *tl->hwsp_seqno, tl->seqno);
457 fence = i915_active_fence_get(&tl->last_request);
458 if (fence) {
459 drm_printf(m, ", engine: %s",
460 to_request(fence)->engine->name);
461 dma_fence_put(fence);
462 }
463 drm_printf(m, " }\n");
464
465 if (show_request) {
466 list_for_each_entry_safe(rq, rn, &tl->requests, link)
467 show_request(m, rq, "", 2);
468 }
469
470 mutex_unlock(&tl->mutex);
471 spin_lock(&timelines->lock);
472
473 /* Resume list iteration after reacquiring spinlock */
474 list_safe_reset_next(tl, tn, link);
475 if (atomic_dec_and_test(&tl->active_count))
476 list_del(&tl->link);
477
478 /* Defer the final release to after the spinlock */
479 if (refcount_dec_and_test(&tl->kref.refcount)) {
480 GEM_BUG_ON(atomic_read(&tl->active_count));
481 list_add(&tl->link, &free);
482 }
483 }
484 spin_unlock(&timelines->lock);
485
486 list_for_each_entry_safe(tl, tn, &free, link)
487 __intel_timeline_free(&tl->kref);
488}
489
490#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
491#include "gt/selftests/mock_timeline.c"
492#include "gt/selftest_timeline.c"
493#endif