Loading...
Note: File does not exist in v3.1.
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2011-2012 Intel Corporation
5 */
6
7/*
8 * This file implements HW context support. On gen5+ a HW context consists of an
9 * opaque GPU object which is referenced at times of context saves and restores.
10 * With RC6 enabled, the context is also referenced as the GPU enters and exists
11 * from RC6 (GPU has it's own internal power context, except on gen5). Though
12 * something like a context does exist for the media ring, the code only
13 * supports contexts for the render ring.
14 *
15 * In software, there is a distinction between contexts created by the user,
16 * and the default HW context. The default HW context is used by GPU clients
17 * that do not request setup of their own hardware context. The default
18 * context's state is never restored to help prevent programming errors. This
19 * would happen if a client ran and piggy-backed off another clients GPU state.
20 * The default context only exists to give the GPU some offset to load as the
21 * current to invoke a save of the context we actually care about. In fact, the
22 * code could likely be constructed, albeit in a more complicated fashion, to
23 * never use the default context, though that limits the driver's ability to
24 * swap out, and/or destroy other contexts.
25 *
26 * All other contexts are created as a request by the GPU client. These contexts
27 * store GPU state, and thus allow GPU clients to not re-emit state (and
28 * potentially query certain state) at any time. The kernel driver makes
29 * certain that the appropriate commands are inserted.
30 *
31 * The context life cycle is semi-complicated in that context BOs may live
32 * longer than the context itself because of the way the hardware, and object
33 * tracking works. Below is a very crude representation of the state machine
34 * describing the context life.
35 * refcount pincount active
36 * S0: initial state 0 0 0
37 * S1: context created 1 0 0
38 * S2: context is currently running 2 1 X
39 * S3: GPU referenced, but not current 2 0 1
40 * S4: context is current, but destroyed 1 1 0
41 * S5: like S3, but destroyed 1 0 1
42 *
43 * The most common (but not all) transitions:
44 * S0->S1: client creates a context
45 * S1->S2: client submits execbuf with context
46 * S2->S3: other clients submits execbuf with context
47 * S3->S1: context object was retired
48 * S3->S2: clients submits another execbuf
49 * S2->S4: context destroy called with current context
50 * S3->S5->S0: destroy path
51 * S4->S5->S0: destroy path on current context
52 *
53 * There are two confusing terms used above:
54 * The "current context" means the context which is currently running on the
55 * GPU. The GPU has loaded its state already and has stored away the gtt
56 * offset of the BO. The GPU is not actively referencing the data at this
57 * offset, but it will on the next context switch. The only way to avoid this
58 * is to do a GPU reset.
59 *
60 * An "active context' is one which was previously the "current context" and is
61 * on the active list waiting for the next context switch to occur. Until this
62 * happens, the object must remain at the same gtt offset. It is therefore
63 * possible to destroy a context, but it is still active.
64 *
65 */
66
67#include <linux/log2.h>
68#include <linux/nospec.h>
69
70#include "gt/gen6_ppgtt.h"
71#include "gt/intel_context.h"
72#include "gt/intel_context_param.h"
73#include "gt/intel_engine_heartbeat.h"
74#include "gt/intel_engine_user.h"
75#include "gt/intel_ring.h"
76
77#include "i915_gem_context.h"
78#include "i915_globals.h"
79#include "i915_trace.h"
80#include "i915_user_extensions.h"
81
82#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
83
84static struct i915_global_gem_context {
85 struct i915_global base;
86 struct kmem_cache *slab_luts;
87} global;
88
89struct i915_lut_handle *i915_lut_handle_alloc(void)
90{
91 return kmem_cache_alloc(global.slab_luts, GFP_KERNEL);
92}
93
94void i915_lut_handle_free(struct i915_lut_handle *lut)
95{
96 return kmem_cache_free(global.slab_luts, lut);
97}
98
99static void lut_close(struct i915_gem_context *ctx)
100{
101 struct radix_tree_iter iter;
102 void __rcu **slot;
103
104 mutex_lock(&ctx->lut_mutex);
105 rcu_read_lock();
106 radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) {
107 struct i915_vma *vma = rcu_dereference_raw(*slot);
108 struct drm_i915_gem_object *obj = vma->obj;
109 struct i915_lut_handle *lut;
110
111 if (!kref_get_unless_zero(&obj->base.refcount))
112 continue;
113
114 spin_lock(&obj->lut_lock);
115 list_for_each_entry(lut, &obj->lut_list, obj_link) {
116 if (lut->ctx != ctx)
117 continue;
118
119 if (lut->handle != iter.index)
120 continue;
121
122 list_del(&lut->obj_link);
123 break;
124 }
125 spin_unlock(&obj->lut_lock);
126
127 if (&lut->obj_link != &obj->lut_list) {
128 i915_lut_handle_free(lut);
129 radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
130 i915_vma_close(vma);
131 i915_gem_object_put(obj);
132 }
133
134 i915_gem_object_put(obj);
135 }
136 rcu_read_unlock();
137 mutex_unlock(&ctx->lut_mutex);
138}
139
140static struct intel_context *
141lookup_user_engine(struct i915_gem_context *ctx,
142 unsigned long flags,
143 const struct i915_engine_class_instance *ci)
144#define LOOKUP_USER_INDEX BIT(0)
145{
146 int idx;
147
148 if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx))
149 return ERR_PTR(-EINVAL);
150
151 if (!i915_gem_context_user_engines(ctx)) {
152 struct intel_engine_cs *engine;
153
154 engine = intel_engine_lookup_user(ctx->i915,
155 ci->engine_class,
156 ci->engine_instance);
157 if (!engine)
158 return ERR_PTR(-EINVAL);
159
160 idx = engine->legacy_idx;
161 } else {
162 idx = ci->engine_instance;
163 }
164
165 return i915_gem_context_get_engine(ctx, idx);
166}
167
168static struct i915_address_space *
169context_get_vm_rcu(struct i915_gem_context *ctx)
170{
171 GEM_BUG_ON(!rcu_access_pointer(ctx->vm));
172
173 do {
174 struct i915_address_space *vm;
175
176 /*
177 * We do not allow downgrading from full-ppgtt [to a shared
178 * global gtt], so ctx->vm cannot become NULL.
179 */
180 vm = rcu_dereference(ctx->vm);
181 if (!kref_get_unless_zero(&vm->ref))
182 continue;
183
184 /*
185 * This ppgtt may have be reallocated between
186 * the read and the kref, and reassigned to a third
187 * context. In order to avoid inadvertent sharing
188 * of this ppgtt with that third context (and not
189 * src), we have to confirm that we have the same
190 * ppgtt after passing through the strong memory
191 * barrier implied by a successful
192 * kref_get_unless_zero().
193 *
194 * Once we have acquired the current ppgtt of ctx,
195 * we no longer care if it is released from ctx, as
196 * it cannot be reallocated elsewhere.
197 */
198
199 if (vm == rcu_access_pointer(ctx->vm))
200 return rcu_pointer_handoff(vm);
201
202 i915_vm_put(vm);
203 } while (1);
204}
205
206static void intel_context_set_gem(struct intel_context *ce,
207 struct i915_gem_context *ctx)
208{
209 GEM_BUG_ON(rcu_access_pointer(ce->gem_context));
210 RCU_INIT_POINTER(ce->gem_context, ctx);
211
212 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))
213 ce->ring = __intel_context_ring_size(SZ_16K);
214
215 if (rcu_access_pointer(ctx->vm)) {
216 struct i915_address_space *vm;
217
218 rcu_read_lock();
219 vm = context_get_vm_rcu(ctx); /* hmm */
220 rcu_read_unlock();
221
222 i915_vm_put(ce->vm);
223 ce->vm = vm;
224 }
225
226 GEM_BUG_ON(ce->timeline);
227 if (ctx->timeline)
228 ce->timeline = intel_timeline_get(ctx->timeline);
229
230 if (ctx->sched.priority >= I915_PRIORITY_NORMAL &&
231 intel_engine_has_timeslices(ce->engine))
232 __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags);
233}
234
235static void __free_engines(struct i915_gem_engines *e, unsigned int count)
236{
237 while (count--) {
238 if (!e->engines[count])
239 continue;
240
241 intel_context_put(e->engines[count]);
242 }
243 kfree(e);
244}
245
246static void free_engines(struct i915_gem_engines *e)
247{
248 __free_engines(e, e->num_engines);
249}
250
251static void free_engines_rcu(struct rcu_head *rcu)
252{
253 struct i915_gem_engines *engines =
254 container_of(rcu, struct i915_gem_engines, rcu);
255
256 i915_sw_fence_fini(&engines->fence);
257 free_engines(engines);
258}
259
260static int __i915_sw_fence_call
261engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
262{
263 struct i915_gem_engines *engines =
264 container_of(fence, typeof(*engines), fence);
265
266 switch (state) {
267 case FENCE_COMPLETE:
268 if (!list_empty(&engines->link)) {
269 struct i915_gem_context *ctx = engines->ctx;
270 unsigned long flags;
271
272 spin_lock_irqsave(&ctx->stale.lock, flags);
273 list_del(&engines->link);
274 spin_unlock_irqrestore(&ctx->stale.lock, flags);
275 }
276 i915_gem_context_put(engines->ctx);
277 break;
278
279 case FENCE_FREE:
280 init_rcu_head(&engines->rcu);
281 call_rcu(&engines->rcu, free_engines_rcu);
282 break;
283 }
284
285 return NOTIFY_DONE;
286}
287
288static struct i915_gem_engines *alloc_engines(unsigned int count)
289{
290 struct i915_gem_engines *e;
291
292 e = kzalloc(struct_size(e, engines, count), GFP_KERNEL);
293 if (!e)
294 return NULL;
295
296 i915_sw_fence_init(&e->fence, engines_notify);
297 return e;
298}
299
300static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx)
301{
302 const struct intel_gt *gt = &ctx->i915->gt;
303 struct intel_engine_cs *engine;
304 struct i915_gem_engines *e;
305 enum intel_engine_id id;
306
307 e = alloc_engines(I915_NUM_ENGINES);
308 if (!e)
309 return ERR_PTR(-ENOMEM);
310
311 for_each_engine(engine, gt, id) {
312 struct intel_context *ce;
313
314 if (engine->legacy_idx == INVALID_ENGINE)
315 continue;
316
317 GEM_BUG_ON(engine->legacy_idx >= I915_NUM_ENGINES);
318 GEM_BUG_ON(e->engines[engine->legacy_idx]);
319
320 ce = intel_context_create(engine);
321 if (IS_ERR(ce)) {
322 __free_engines(e, e->num_engines + 1);
323 return ERR_CAST(ce);
324 }
325
326 intel_context_set_gem(ce, ctx);
327
328 e->engines[engine->legacy_idx] = ce;
329 e->num_engines = max(e->num_engines, engine->legacy_idx);
330 }
331 e->num_engines++;
332
333 return e;
334}
335
336static void i915_gem_context_free(struct i915_gem_context *ctx)
337{
338 GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
339
340 spin_lock(&ctx->i915->gem.contexts.lock);
341 list_del(&ctx->link);
342 spin_unlock(&ctx->i915->gem.contexts.lock);
343
344 mutex_destroy(&ctx->engines_mutex);
345 mutex_destroy(&ctx->lut_mutex);
346
347 if (ctx->timeline)
348 intel_timeline_put(ctx->timeline);
349
350 put_pid(ctx->pid);
351 mutex_destroy(&ctx->mutex);
352
353 kfree_rcu(ctx, rcu);
354}
355
356static void contexts_free_all(struct llist_node *list)
357{
358 struct i915_gem_context *ctx, *cn;
359
360 llist_for_each_entry_safe(ctx, cn, list, free_link)
361 i915_gem_context_free(ctx);
362}
363
364static void contexts_flush_free(struct i915_gem_contexts *gc)
365{
366 contexts_free_all(llist_del_all(&gc->free_list));
367}
368
369static void contexts_free_worker(struct work_struct *work)
370{
371 struct i915_gem_contexts *gc =
372 container_of(work, typeof(*gc), free_work);
373
374 contexts_flush_free(gc);
375}
376
377void i915_gem_context_release(struct kref *ref)
378{
379 struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref);
380 struct i915_gem_contexts *gc = &ctx->i915->gem.contexts;
381
382 trace_i915_context_free(ctx);
383 if (llist_add(&ctx->free_link, &gc->free_list))
384 schedule_work(&gc->free_work);
385}
386
387static inline struct i915_gem_engines *
388__context_engines_static(const struct i915_gem_context *ctx)
389{
390 return rcu_dereference_protected(ctx->engines, true);
391}
392
393static bool __reset_engine(struct intel_engine_cs *engine)
394{
395 struct intel_gt *gt = engine->gt;
396 bool success = false;
397
398 if (!intel_has_reset_engine(gt))
399 return false;
400
401 if (!test_and_set_bit(I915_RESET_ENGINE + engine->id,
402 >->reset.flags)) {
403 success = intel_engine_reset(engine, NULL) == 0;
404 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
405 >->reset.flags);
406 }
407
408 return success;
409}
410
411static void __reset_context(struct i915_gem_context *ctx,
412 struct intel_engine_cs *engine)
413{
414 intel_gt_handle_error(engine->gt, engine->mask, 0,
415 "context closure in %s", ctx->name);
416}
417
418static bool __cancel_engine(struct intel_engine_cs *engine)
419{
420 /*
421 * Send a "high priority pulse" down the engine to cause the
422 * current request to be momentarily preempted. (If it fails to
423 * be preempted, it will be reset). As we have marked our context
424 * as banned, any incomplete request, including any running, will
425 * be skipped following the preemption.
426 *
427 * If there is no hangchecking (one of the reasons why we try to
428 * cancel the context) and no forced preemption, there may be no
429 * means by which we reset the GPU and evict the persistent hog.
430 * Ergo if we are unable to inject a preemptive pulse that can
431 * kill the banned context, we fallback to doing a local reset
432 * instead.
433 */
434 if (IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT) &&
435 !intel_engine_pulse(engine))
436 return true;
437
438 /* If we are unable to send a pulse, try resetting this engine. */
439 return __reset_engine(engine);
440}
441
442static bool
443__active_engine(struct i915_request *rq, struct intel_engine_cs **active)
444{
445 struct intel_engine_cs *engine, *locked;
446 bool ret = false;
447
448 /*
449 * Serialise with __i915_request_submit() so that it sees
450 * is-banned?, or we know the request is already inflight.
451 *
452 * Note that rq->engine is unstable, and so we double
453 * check that we have acquired the lock on the final engine.
454 */
455 locked = READ_ONCE(rq->engine);
456 spin_lock_irq(&locked->active.lock);
457 while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
458 spin_unlock(&locked->active.lock);
459 locked = engine;
460 spin_lock(&locked->active.lock);
461 }
462
463 if (!i915_request_completed(rq)) {
464 if (i915_request_is_active(rq) && rq->fence.error != -EIO)
465 *active = locked;
466 ret = true;
467 }
468
469 spin_unlock_irq(&locked->active.lock);
470
471 return ret;
472}
473
474static struct intel_engine_cs *active_engine(struct intel_context *ce)
475{
476 struct intel_engine_cs *engine = NULL;
477 struct i915_request *rq;
478
479 if (!ce->timeline)
480 return NULL;
481
482 rcu_read_lock();
483 list_for_each_entry_rcu(rq, &ce->timeline->requests, link) {
484 if (i915_request_is_active(rq) && i915_request_completed(rq))
485 continue;
486
487 /* Check with the backend if the request is inflight */
488 if (__active_engine(rq, &engine))
489 break;
490 }
491 rcu_read_unlock();
492
493 return engine;
494}
495
496static void kill_engines(struct i915_gem_engines *engines)
497{
498 struct i915_gem_engines_iter it;
499 struct intel_context *ce;
500
501 /*
502 * Map the user's engine back to the actual engines; one virtual
503 * engine will be mapped to multiple engines, and using ctx->engine[]
504 * the same engine may be have multiple instances in the user's map.
505 * However, we only care about pending requests, so only include
506 * engines on which there are incomplete requests.
507 */
508 for_each_gem_engine(ce, engines, it) {
509 struct intel_engine_cs *engine;
510
511 if (intel_context_set_banned(ce))
512 continue;
513
514 /*
515 * Check the current active state of this context; if we
516 * are currently executing on the GPU we need to evict
517 * ourselves. On the other hand, if we haven't yet been
518 * submitted to the GPU or if everything is complete,
519 * we have nothing to do.
520 */
521 engine = active_engine(ce);
522
523 /* First attempt to gracefully cancel the context */
524 if (engine && !__cancel_engine(engine))
525 /*
526 * If we are unable to send a preemptive pulse to bump
527 * the context from the GPU, we have to resort to a full
528 * reset. We hope the collateral damage is worth it.
529 */
530 __reset_context(engines->ctx, engine);
531 }
532}
533
534static void kill_stale_engines(struct i915_gem_context *ctx)
535{
536 struct i915_gem_engines *pos, *next;
537
538 spin_lock_irq(&ctx->stale.lock);
539 GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
540 list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) {
541 if (!i915_sw_fence_await(&pos->fence)) {
542 list_del_init(&pos->link);
543 continue;
544 }
545
546 spin_unlock_irq(&ctx->stale.lock);
547
548 kill_engines(pos);
549
550 spin_lock_irq(&ctx->stale.lock);
551 GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence));
552 list_safe_reset_next(pos, next, link);
553 list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */
554
555 i915_sw_fence_complete(&pos->fence);
556 }
557 spin_unlock_irq(&ctx->stale.lock);
558}
559
560static void kill_context(struct i915_gem_context *ctx)
561{
562 kill_stale_engines(ctx);
563}
564
565static void engines_idle_release(struct i915_gem_context *ctx,
566 struct i915_gem_engines *engines)
567{
568 struct i915_gem_engines_iter it;
569 struct intel_context *ce;
570
571 INIT_LIST_HEAD(&engines->link);
572
573 engines->ctx = i915_gem_context_get(ctx);
574
575 for_each_gem_engine(ce, engines, it) {
576 int err;
577
578 /* serialises with execbuf */
579 set_bit(CONTEXT_CLOSED_BIT, &ce->flags);
580 if (!intel_context_pin_if_active(ce))
581 continue;
582
583 /* Wait until context is finally scheduled out and retired */
584 err = i915_sw_fence_await_active(&engines->fence,
585 &ce->active,
586 I915_ACTIVE_AWAIT_BARRIER);
587 intel_context_unpin(ce);
588 if (err)
589 goto kill;
590 }
591
592 spin_lock_irq(&ctx->stale.lock);
593 if (!i915_gem_context_is_closed(ctx))
594 list_add_tail(&engines->link, &ctx->stale.engines);
595 spin_unlock_irq(&ctx->stale.lock);
596
597kill:
598 if (list_empty(&engines->link)) /* raced, already closed */
599 kill_engines(engines);
600
601 i915_sw_fence_commit(&engines->fence);
602}
603
604static void set_closed_name(struct i915_gem_context *ctx)
605{
606 char *s;
607
608 /* Replace '[]' with '<>' to indicate closed in debug prints */
609
610 s = strrchr(ctx->name, '[');
611 if (!s)
612 return;
613
614 *s = '<';
615
616 s = strchr(s + 1, ']');
617 if (s)
618 *s = '>';
619}
620
621static void context_close(struct i915_gem_context *ctx)
622{
623 struct i915_address_space *vm;
624
625 /* Flush any concurrent set_engines() */
626 mutex_lock(&ctx->engines_mutex);
627 engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1));
628 i915_gem_context_set_closed(ctx);
629 mutex_unlock(&ctx->engines_mutex);
630
631 mutex_lock(&ctx->mutex);
632
633 set_closed_name(ctx);
634
635 vm = i915_gem_context_vm(ctx);
636 if (vm)
637 i915_vm_close(vm);
638
639 ctx->file_priv = ERR_PTR(-EBADF);
640
641 /*
642 * The LUT uses the VMA as a backpointer to unref the object,
643 * so we need to clear the LUT before we close all the VMA (inside
644 * the ppgtt).
645 */
646 lut_close(ctx);
647
648 mutex_unlock(&ctx->mutex);
649
650 /*
651 * If the user has disabled hangchecking, we can not be sure that
652 * the batches will ever complete after the context is closed,
653 * keeping the context and all resources pinned forever. So in this
654 * case we opt to forcibly kill off all remaining requests on
655 * context close.
656 */
657 if (!i915_gem_context_is_persistent(ctx) ||
658 !ctx->i915->params.enable_hangcheck)
659 kill_context(ctx);
660
661 i915_gem_context_put(ctx);
662}
663
664static int __context_set_persistence(struct i915_gem_context *ctx, bool state)
665{
666 if (i915_gem_context_is_persistent(ctx) == state)
667 return 0;
668
669 if (state) {
670 /*
671 * Only contexts that are short-lived [that will expire or be
672 * reset] are allowed to survive past termination. We require
673 * hangcheck to ensure that the persistent requests are healthy.
674 */
675 if (!ctx->i915->params.enable_hangcheck)
676 return -EINVAL;
677
678 i915_gem_context_set_persistence(ctx);
679 } else {
680 /* To cancel a context we use "preempt-to-idle" */
681 if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
682 return -ENODEV;
683
684 /*
685 * If the cancel fails, we then need to reset, cleanly!
686 *
687 * If the per-engine reset fails, all hope is lost! We resort
688 * to a full GPU reset in that unlikely case, but realistically
689 * if the engine could not reset, the full reset does not fare
690 * much better. The damage has been done.
691 *
692 * However, if we cannot reset an engine by itself, we cannot
693 * cleanup a hanging persistent context without causing
694 * colateral damage, and we should not pretend we can by
695 * exposing the interface.
696 */
697 if (!intel_has_reset_engine(&ctx->i915->gt))
698 return -ENODEV;
699
700 i915_gem_context_clear_persistence(ctx);
701 }
702
703 return 0;
704}
705
706static struct i915_gem_context *
707__create_context(struct drm_i915_private *i915)
708{
709 struct i915_gem_context *ctx;
710 struct i915_gem_engines *e;
711 int err;
712 int i;
713
714 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
715 if (!ctx)
716 return ERR_PTR(-ENOMEM);
717
718 kref_init(&ctx->ref);
719 ctx->i915 = i915;
720 ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL);
721 mutex_init(&ctx->mutex);
722 INIT_LIST_HEAD(&ctx->link);
723
724 spin_lock_init(&ctx->stale.lock);
725 INIT_LIST_HEAD(&ctx->stale.engines);
726
727 mutex_init(&ctx->engines_mutex);
728 e = default_engines(ctx);
729 if (IS_ERR(e)) {
730 err = PTR_ERR(e);
731 goto err_free;
732 }
733 RCU_INIT_POINTER(ctx->engines, e);
734
735 INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
736 mutex_init(&ctx->lut_mutex);
737
738 /* NB: Mark all slices as needing a remap so that when the context first
739 * loads it will restore whatever remap state already exists. If there
740 * is no remap info, it will be a NOP. */
741 ctx->remap_slice = ALL_L3_SLICES(i915);
742
743 i915_gem_context_set_bannable(ctx);
744 i915_gem_context_set_recoverable(ctx);
745 __context_set_persistence(ctx, true /* cgroup hook? */);
746
747 for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
748 ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
749
750 return ctx;
751
752err_free:
753 kfree(ctx);
754 return ERR_PTR(err);
755}
756
757static inline struct i915_gem_engines *
758__context_engines_await(const struct i915_gem_context *ctx)
759{
760 struct i915_gem_engines *engines;
761
762 rcu_read_lock();
763 do {
764 engines = rcu_dereference(ctx->engines);
765 GEM_BUG_ON(!engines);
766
767 if (unlikely(!i915_sw_fence_await(&engines->fence)))
768 continue;
769
770 if (likely(engines == rcu_access_pointer(ctx->engines)))
771 break;
772
773 i915_sw_fence_complete(&engines->fence);
774 } while (1);
775 rcu_read_unlock();
776
777 return engines;
778}
779
780static int
781context_apply_all(struct i915_gem_context *ctx,
782 int (*fn)(struct intel_context *ce, void *data),
783 void *data)
784{
785 struct i915_gem_engines_iter it;
786 struct i915_gem_engines *e;
787 struct intel_context *ce;
788 int err = 0;
789
790 e = __context_engines_await(ctx);
791 for_each_gem_engine(ce, e, it) {
792 err = fn(ce, data);
793 if (err)
794 break;
795 }
796 i915_sw_fence_complete(&e->fence);
797
798 return err;
799}
800
801static int __apply_ppgtt(struct intel_context *ce, void *vm)
802{
803 i915_vm_put(ce->vm);
804 ce->vm = i915_vm_get(vm);
805 return 0;
806}
807
808static struct i915_address_space *
809__set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm)
810{
811 struct i915_address_space *old;
812
813 old = rcu_replace_pointer(ctx->vm,
814 i915_vm_open(vm),
815 lockdep_is_held(&ctx->mutex));
816 GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old));
817
818 context_apply_all(ctx, __apply_ppgtt, vm);
819
820 return old;
821}
822
823static void __assign_ppgtt(struct i915_gem_context *ctx,
824 struct i915_address_space *vm)
825{
826 if (vm == rcu_access_pointer(ctx->vm))
827 return;
828
829 vm = __set_ppgtt(ctx, vm);
830 if (vm)
831 i915_vm_close(vm);
832}
833
834static void __set_timeline(struct intel_timeline **dst,
835 struct intel_timeline *src)
836{
837 struct intel_timeline *old = *dst;
838
839 *dst = src ? intel_timeline_get(src) : NULL;
840
841 if (old)
842 intel_timeline_put(old);
843}
844
845static int __apply_timeline(struct intel_context *ce, void *timeline)
846{
847 __set_timeline(&ce->timeline, timeline);
848 return 0;
849}
850
851static void __assign_timeline(struct i915_gem_context *ctx,
852 struct intel_timeline *timeline)
853{
854 __set_timeline(&ctx->timeline, timeline);
855 context_apply_all(ctx, __apply_timeline, timeline);
856}
857
858static struct i915_gem_context *
859i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags)
860{
861 struct i915_gem_context *ctx;
862
863 if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE &&
864 !HAS_EXECLISTS(i915))
865 return ERR_PTR(-EINVAL);
866
867 /* Reap the stale contexts */
868 contexts_flush_free(&i915->gem.contexts);
869
870 ctx = __create_context(i915);
871 if (IS_ERR(ctx))
872 return ctx;
873
874 if (HAS_FULL_PPGTT(i915)) {
875 struct i915_ppgtt *ppgtt;
876
877 ppgtt = i915_ppgtt_create(&i915->gt);
878 if (IS_ERR(ppgtt)) {
879 drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n",
880 PTR_ERR(ppgtt));
881 context_close(ctx);
882 return ERR_CAST(ppgtt);
883 }
884
885 mutex_lock(&ctx->mutex);
886 __assign_ppgtt(ctx, &ppgtt->vm);
887 mutex_unlock(&ctx->mutex);
888
889 i915_vm_put(&ppgtt->vm);
890 }
891
892 if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) {
893 struct intel_timeline *timeline;
894
895 timeline = intel_timeline_create(&i915->gt, NULL);
896 if (IS_ERR(timeline)) {
897 context_close(ctx);
898 return ERR_CAST(timeline);
899 }
900
901 __assign_timeline(ctx, timeline);
902 intel_timeline_put(timeline);
903 }
904
905 trace_i915_context_create(ctx);
906
907 return ctx;
908}
909
910static void init_contexts(struct i915_gem_contexts *gc)
911{
912 spin_lock_init(&gc->lock);
913 INIT_LIST_HEAD(&gc->list);
914
915 INIT_WORK(&gc->free_work, contexts_free_worker);
916 init_llist_head(&gc->free_list);
917}
918
919void i915_gem_init__contexts(struct drm_i915_private *i915)
920{
921 init_contexts(&i915->gem.contexts);
922 drm_dbg(&i915->drm, "%s context support initialized\n",
923 DRIVER_CAPS(i915)->has_logical_contexts ?
924 "logical" : "fake");
925}
926
927void i915_gem_driver_release__contexts(struct drm_i915_private *i915)
928{
929 flush_work(&i915->gem.contexts.free_work);
930 rcu_barrier(); /* and flush the left over RCU frees */
931}
932
933static int gem_context_register(struct i915_gem_context *ctx,
934 struct drm_i915_file_private *fpriv,
935 u32 *id)
936{
937 struct drm_i915_private *i915 = ctx->i915;
938 struct i915_address_space *vm;
939 int ret;
940
941 ctx->file_priv = fpriv;
942
943 mutex_lock(&ctx->mutex);
944 vm = i915_gem_context_vm(ctx);
945 if (vm)
946 WRITE_ONCE(vm->file, fpriv); /* XXX */
947 mutex_unlock(&ctx->mutex);
948
949 ctx->pid = get_task_pid(current, PIDTYPE_PID);
950 snprintf(ctx->name, sizeof(ctx->name), "%s[%d]",
951 current->comm, pid_nr(ctx->pid));
952
953 /* And finally expose ourselves to userspace via the idr */
954 ret = xa_alloc(&fpriv->context_xa, id, ctx, xa_limit_32b, GFP_KERNEL);
955 if (ret)
956 goto err_pid;
957
958 spin_lock(&i915->gem.contexts.lock);
959 list_add_tail(&ctx->link, &i915->gem.contexts.list);
960 spin_unlock(&i915->gem.contexts.lock);
961
962 return 0;
963
964err_pid:
965 put_pid(fetch_and_zero(&ctx->pid));
966 return ret;
967}
968
969int i915_gem_context_open(struct drm_i915_private *i915,
970 struct drm_file *file)
971{
972 struct drm_i915_file_private *file_priv = file->driver_priv;
973 struct i915_gem_context *ctx;
974 int err;
975 u32 id;
976
977 xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC);
978
979 /* 0 reserved for invalid/unassigned ppgtt */
980 xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1);
981
982 ctx = i915_gem_create_context(i915, 0);
983 if (IS_ERR(ctx)) {
984 err = PTR_ERR(ctx);
985 goto err;
986 }
987
988 err = gem_context_register(ctx, file_priv, &id);
989 if (err < 0)
990 goto err_ctx;
991
992 GEM_BUG_ON(id);
993 return 0;
994
995err_ctx:
996 context_close(ctx);
997err:
998 xa_destroy(&file_priv->vm_xa);
999 xa_destroy(&file_priv->context_xa);
1000 return err;
1001}
1002
1003void i915_gem_context_close(struct drm_file *file)
1004{
1005 struct drm_i915_file_private *file_priv = file->driver_priv;
1006 struct drm_i915_private *i915 = file_priv->dev_priv;
1007 struct i915_address_space *vm;
1008 struct i915_gem_context *ctx;
1009 unsigned long idx;
1010
1011 xa_for_each(&file_priv->context_xa, idx, ctx)
1012 context_close(ctx);
1013 xa_destroy(&file_priv->context_xa);
1014
1015 xa_for_each(&file_priv->vm_xa, idx, vm)
1016 i915_vm_put(vm);
1017 xa_destroy(&file_priv->vm_xa);
1018
1019 contexts_flush_free(&i915->gem.contexts);
1020}
1021
1022int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data,
1023 struct drm_file *file)
1024{
1025 struct drm_i915_private *i915 = to_i915(dev);
1026 struct drm_i915_gem_vm_control *args = data;
1027 struct drm_i915_file_private *file_priv = file->driver_priv;
1028 struct i915_ppgtt *ppgtt;
1029 u32 id;
1030 int err;
1031
1032 if (!HAS_FULL_PPGTT(i915))
1033 return -ENODEV;
1034
1035 if (args->flags)
1036 return -EINVAL;
1037
1038 ppgtt = i915_ppgtt_create(&i915->gt);
1039 if (IS_ERR(ppgtt))
1040 return PTR_ERR(ppgtt);
1041
1042 ppgtt->vm.file = file_priv;
1043
1044 if (args->extensions) {
1045 err = i915_user_extensions(u64_to_user_ptr(args->extensions),
1046 NULL, 0,
1047 ppgtt);
1048 if (err)
1049 goto err_put;
1050 }
1051
1052 err = xa_alloc(&file_priv->vm_xa, &id, &ppgtt->vm,
1053 xa_limit_32b, GFP_KERNEL);
1054 if (err)
1055 goto err_put;
1056
1057 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */
1058 args->vm_id = id;
1059 return 0;
1060
1061err_put:
1062 i915_vm_put(&ppgtt->vm);
1063 return err;
1064}
1065
1066int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data,
1067 struct drm_file *file)
1068{
1069 struct drm_i915_file_private *file_priv = file->driver_priv;
1070 struct drm_i915_gem_vm_control *args = data;
1071 struct i915_address_space *vm;
1072
1073 if (args->flags)
1074 return -EINVAL;
1075
1076 if (args->extensions)
1077 return -EINVAL;
1078
1079 vm = xa_erase(&file_priv->vm_xa, args->vm_id);
1080 if (!vm)
1081 return -ENOENT;
1082
1083 i915_vm_put(vm);
1084 return 0;
1085}
1086
1087struct context_barrier_task {
1088 struct i915_active base;
1089 void (*task)(void *data);
1090 void *data;
1091};
1092
1093__i915_active_call
1094static void cb_retire(struct i915_active *base)
1095{
1096 struct context_barrier_task *cb = container_of(base, typeof(*cb), base);
1097
1098 if (cb->task)
1099 cb->task(cb->data);
1100
1101 i915_active_fini(&cb->base);
1102 kfree(cb);
1103}
1104
1105I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault);
1106static int context_barrier_task(struct i915_gem_context *ctx,
1107 intel_engine_mask_t engines,
1108 bool (*skip)(struct intel_context *ce, void *data),
1109 int (*emit)(struct i915_request *rq, void *data),
1110 void (*task)(void *data),
1111 void *data)
1112{
1113 struct context_barrier_task *cb;
1114 struct i915_gem_engines_iter it;
1115 struct i915_gem_engines *e;
1116 struct intel_context *ce;
1117 int err = 0;
1118
1119 GEM_BUG_ON(!task);
1120
1121 cb = kmalloc(sizeof(*cb), GFP_KERNEL);
1122 if (!cb)
1123 return -ENOMEM;
1124
1125 i915_active_init(&cb->base, NULL, cb_retire);
1126 err = i915_active_acquire(&cb->base);
1127 if (err) {
1128 kfree(cb);
1129 return err;
1130 }
1131
1132 e = __context_engines_await(ctx);
1133 if (!e) {
1134 i915_active_release(&cb->base);
1135 return -ENOENT;
1136 }
1137
1138 for_each_gem_engine(ce, e, it) {
1139 struct i915_request *rq;
1140
1141 if (I915_SELFTEST_ONLY(context_barrier_inject_fault &
1142 ce->engine->mask)) {
1143 err = -ENXIO;
1144 break;
1145 }
1146
1147 if (!(ce->engine->mask & engines))
1148 continue;
1149
1150 if (skip && skip(ce, data))
1151 continue;
1152
1153 rq = intel_context_create_request(ce);
1154 if (IS_ERR(rq)) {
1155 err = PTR_ERR(rq);
1156 break;
1157 }
1158
1159 err = 0;
1160 if (emit)
1161 err = emit(rq, data);
1162 if (err == 0)
1163 err = i915_active_add_request(&cb->base, rq);
1164
1165 i915_request_add(rq);
1166 if (err)
1167 break;
1168 }
1169 i915_sw_fence_complete(&e->fence);
1170
1171 cb->task = err ? NULL : task; /* caller needs to unwind instead */
1172 cb->data = data;
1173
1174 i915_active_release(&cb->base);
1175
1176 return err;
1177}
1178
1179static int get_ppgtt(struct drm_i915_file_private *file_priv,
1180 struct i915_gem_context *ctx,
1181 struct drm_i915_gem_context_param *args)
1182{
1183 struct i915_address_space *vm;
1184 int err;
1185 u32 id;
1186
1187 if (!rcu_access_pointer(ctx->vm))
1188 return -ENODEV;
1189
1190 rcu_read_lock();
1191 vm = context_get_vm_rcu(ctx);
1192 rcu_read_unlock();
1193 if (!vm)
1194 return -ENODEV;
1195
1196 err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL);
1197 if (err)
1198 goto err_put;
1199
1200 i915_vm_open(vm);
1201
1202 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */
1203 args->value = id;
1204 args->size = 0;
1205
1206err_put:
1207 i915_vm_put(vm);
1208 return err;
1209}
1210
1211static void set_ppgtt_barrier(void *data)
1212{
1213 struct i915_address_space *old = data;
1214
1215 if (INTEL_GEN(old->i915) < 8)
1216 gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old));
1217
1218 i915_vm_close(old);
1219}
1220
1221static int emit_ppgtt_update(struct i915_request *rq, void *data)
1222{
1223 struct i915_address_space *vm = rq->context->vm;
1224 struct intel_engine_cs *engine = rq->engine;
1225 u32 base = engine->mmio_base;
1226 u32 *cs;
1227 int i;
1228
1229 if (i915_vm_is_4lvl(vm)) {
1230 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1231 const dma_addr_t pd_daddr = px_dma(ppgtt->pd);
1232
1233 cs = intel_ring_begin(rq, 6);
1234 if (IS_ERR(cs))
1235 return PTR_ERR(cs);
1236
1237 *cs++ = MI_LOAD_REGISTER_IMM(2);
1238
1239 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0));
1240 *cs++ = upper_32_bits(pd_daddr);
1241 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0));
1242 *cs++ = lower_32_bits(pd_daddr);
1243
1244 *cs++ = MI_NOOP;
1245 intel_ring_advance(rq, cs);
1246 } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) {
1247 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1248 int err;
1249
1250 /* Magic required to prevent forcewake errors! */
1251 err = engine->emit_flush(rq, EMIT_INVALIDATE);
1252 if (err)
1253 return err;
1254
1255 cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
1256 if (IS_ERR(cs))
1257 return PTR_ERR(cs);
1258
1259 *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
1260 for (i = GEN8_3LVL_PDPES; i--; ) {
1261 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
1262
1263 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
1264 *cs++ = upper_32_bits(pd_daddr);
1265 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
1266 *cs++ = lower_32_bits(pd_daddr);
1267 }
1268 *cs++ = MI_NOOP;
1269 intel_ring_advance(rq, cs);
1270 }
1271
1272 return 0;
1273}
1274
1275static bool skip_ppgtt_update(struct intel_context *ce, void *data)
1276{
1277 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))
1278 return true;
1279
1280 if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915))
1281 return false;
1282
1283 if (!atomic_read(&ce->pin_count))
1284 return true;
1285
1286 /* ppGTT is not part of the legacy context image */
1287 if (gen6_ppgtt_pin(i915_vm_to_ppgtt(ce->vm)))
1288 return true;
1289
1290 return false;
1291}
1292
1293static int set_ppgtt(struct drm_i915_file_private *file_priv,
1294 struct i915_gem_context *ctx,
1295 struct drm_i915_gem_context_param *args)
1296{
1297 struct i915_address_space *vm, *old;
1298 int err;
1299
1300 if (args->size)
1301 return -EINVAL;
1302
1303 if (!rcu_access_pointer(ctx->vm))
1304 return -ENODEV;
1305
1306 if (upper_32_bits(args->value))
1307 return -ENOENT;
1308
1309 rcu_read_lock();
1310 vm = xa_load(&file_priv->vm_xa, args->value);
1311 if (vm && !kref_get_unless_zero(&vm->ref))
1312 vm = NULL;
1313 rcu_read_unlock();
1314 if (!vm)
1315 return -ENOENT;
1316
1317 err = mutex_lock_interruptible(&ctx->mutex);
1318 if (err)
1319 goto out;
1320
1321 if (i915_gem_context_is_closed(ctx)) {
1322 err = -ENOENT;
1323 goto unlock;
1324 }
1325
1326 if (vm == rcu_access_pointer(ctx->vm))
1327 goto unlock;
1328
1329 old = __set_ppgtt(ctx, vm);
1330
1331 /* Teardown the existing obj:vma cache, it will have to be rebuilt. */
1332 lut_close(ctx);
1333
1334 /*
1335 * We need to flush any requests using the current ppgtt before
1336 * we release it as the requests do not hold a reference themselves,
1337 * only indirectly through the context.
1338 */
1339 err = context_barrier_task(ctx, ALL_ENGINES,
1340 skip_ppgtt_update,
1341 emit_ppgtt_update,
1342 set_ppgtt_barrier,
1343 old);
1344 if (err) {
1345 i915_vm_close(__set_ppgtt(ctx, old));
1346 i915_vm_close(old);
1347 lut_close(ctx); /* force a rebuild of the old obj:vma cache */
1348 }
1349
1350unlock:
1351 mutex_unlock(&ctx->mutex);
1352out:
1353 i915_vm_put(vm);
1354 return err;
1355}
1356
1357static int __apply_ringsize(struct intel_context *ce, void *sz)
1358{
1359 return intel_context_set_ring_size(ce, (unsigned long)sz);
1360}
1361
1362static int set_ringsize(struct i915_gem_context *ctx,
1363 struct drm_i915_gem_context_param *args)
1364{
1365 if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915))
1366 return -ENODEV;
1367
1368 if (args->size)
1369 return -EINVAL;
1370
1371 if (!IS_ALIGNED(args->value, I915_GTT_PAGE_SIZE))
1372 return -EINVAL;
1373
1374 if (args->value < I915_GTT_PAGE_SIZE)
1375 return -EINVAL;
1376
1377 if (args->value > 128 * I915_GTT_PAGE_SIZE)
1378 return -EINVAL;
1379
1380 return context_apply_all(ctx,
1381 __apply_ringsize,
1382 __intel_context_ring_size(args->value));
1383}
1384
1385static int __get_ringsize(struct intel_context *ce, void *arg)
1386{
1387 long sz;
1388
1389 sz = intel_context_get_ring_size(ce);
1390 GEM_BUG_ON(sz > INT_MAX);
1391
1392 return sz; /* stop on first engine */
1393}
1394
1395static int get_ringsize(struct i915_gem_context *ctx,
1396 struct drm_i915_gem_context_param *args)
1397{
1398 int sz;
1399
1400 if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915))
1401 return -ENODEV;
1402
1403 if (args->size)
1404 return -EINVAL;
1405
1406 sz = context_apply_all(ctx, __get_ringsize, NULL);
1407 if (sz < 0)
1408 return sz;
1409
1410 args->value = sz;
1411 return 0;
1412}
1413
1414int
1415i915_gem_user_to_context_sseu(struct intel_gt *gt,
1416 const struct drm_i915_gem_context_param_sseu *user,
1417 struct intel_sseu *context)
1418{
1419 const struct sseu_dev_info *device = >->info.sseu;
1420 struct drm_i915_private *i915 = gt->i915;
1421
1422 /* No zeros in any field. */
1423 if (!user->slice_mask || !user->subslice_mask ||
1424 !user->min_eus_per_subslice || !user->max_eus_per_subslice)
1425 return -EINVAL;
1426
1427 /* Max > min. */
1428 if (user->max_eus_per_subslice < user->min_eus_per_subslice)
1429 return -EINVAL;
1430
1431 /*
1432 * Some future proofing on the types since the uAPI is wider than the
1433 * current internal implementation.
1434 */
1435 if (overflows_type(user->slice_mask, context->slice_mask) ||
1436 overflows_type(user->subslice_mask, context->subslice_mask) ||
1437 overflows_type(user->min_eus_per_subslice,
1438 context->min_eus_per_subslice) ||
1439 overflows_type(user->max_eus_per_subslice,
1440 context->max_eus_per_subslice))
1441 return -EINVAL;
1442
1443 /* Check validity against hardware. */
1444 if (user->slice_mask & ~device->slice_mask)
1445 return -EINVAL;
1446
1447 if (user->subslice_mask & ~device->subslice_mask[0])
1448 return -EINVAL;
1449
1450 if (user->max_eus_per_subslice > device->max_eus_per_subslice)
1451 return -EINVAL;
1452
1453 context->slice_mask = user->slice_mask;
1454 context->subslice_mask = user->subslice_mask;
1455 context->min_eus_per_subslice = user->min_eus_per_subslice;
1456 context->max_eus_per_subslice = user->max_eus_per_subslice;
1457
1458 /* Part specific restrictions. */
1459 if (IS_GEN(i915, 11)) {
1460 unsigned int hw_s = hweight8(device->slice_mask);
1461 unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]);
1462 unsigned int req_s = hweight8(context->slice_mask);
1463 unsigned int req_ss = hweight8(context->subslice_mask);
1464
1465 /*
1466 * Only full subslice enablement is possible if more than one
1467 * slice is turned on.
1468 */
1469 if (req_s > 1 && req_ss != hw_ss_per_s)
1470 return -EINVAL;
1471
1472 /*
1473 * If more than four (SScount bitfield limit) subslices are
1474 * requested then the number has to be even.
1475 */
1476 if (req_ss > 4 && (req_ss & 1))
1477 return -EINVAL;
1478
1479 /*
1480 * If only one slice is enabled and subslice count is below the
1481 * device full enablement, it must be at most half of the all
1482 * available subslices.
1483 */
1484 if (req_s == 1 && req_ss < hw_ss_per_s &&
1485 req_ss > (hw_ss_per_s / 2))
1486 return -EINVAL;
1487
1488 /* ABI restriction - VME use case only. */
1489
1490 /* All slices or one slice only. */
1491 if (req_s != 1 && req_s != hw_s)
1492 return -EINVAL;
1493
1494 /*
1495 * Half subslices or full enablement only when one slice is
1496 * enabled.
1497 */
1498 if (req_s == 1 &&
1499 (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2)))
1500 return -EINVAL;
1501
1502 /* No EU configuration changes. */
1503 if ((user->min_eus_per_subslice !=
1504 device->max_eus_per_subslice) ||
1505 (user->max_eus_per_subslice !=
1506 device->max_eus_per_subslice))
1507 return -EINVAL;
1508 }
1509
1510 return 0;
1511}
1512
1513static int set_sseu(struct i915_gem_context *ctx,
1514 struct drm_i915_gem_context_param *args)
1515{
1516 struct drm_i915_private *i915 = ctx->i915;
1517 struct drm_i915_gem_context_param_sseu user_sseu;
1518 struct intel_context *ce;
1519 struct intel_sseu sseu;
1520 unsigned long lookup;
1521 int ret;
1522
1523 if (args->size < sizeof(user_sseu))
1524 return -EINVAL;
1525
1526 if (!IS_GEN(i915, 11))
1527 return -ENODEV;
1528
1529 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
1530 sizeof(user_sseu)))
1531 return -EFAULT;
1532
1533 if (user_sseu.rsvd)
1534 return -EINVAL;
1535
1536 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX))
1537 return -EINVAL;
1538
1539 lookup = 0;
1540 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)
1541 lookup |= LOOKUP_USER_INDEX;
1542
1543 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine);
1544 if (IS_ERR(ce))
1545 return PTR_ERR(ce);
1546
1547 /* Only render engine supports RPCS configuration. */
1548 if (ce->engine->class != RENDER_CLASS) {
1549 ret = -ENODEV;
1550 goto out_ce;
1551 }
1552
1553 ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu);
1554 if (ret)
1555 goto out_ce;
1556
1557 ret = intel_context_reconfigure_sseu(ce, sseu);
1558 if (ret)
1559 goto out_ce;
1560
1561 args->size = sizeof(user_sseu);
1562
1563out_ce:
1564 intel_context_put(ce);
1565 return ret;
1566}
1567
1568struct set_engines {
1569 struct i915_gem_context *ctx;
1570 struct i915_gem_engines *engines;
1571};
1572
1573static int
1574set_engines__load_balance(struct i915_user_extension __user *base, void *data)
1575{
1576 struct i915_context_engines_load_balance __user *ext =
1577 container_of_user(base, typeof(*ext), base);
1578 const struct set_engines *set = data;
1579 struct drm_i915_private *i915 = set->ctx->i915;
1580 struct intel_engine_cs *stack[16];
1581 struct intel_engine_cs **siblings;
1582 struct intel_context *ce;
1583 u16 num_siblings, idx;
1584 unsigned int n;
1585 int err;
1586
1587 if (!HAS_EXECLISTS(i915))
1588 return -ENODEV;
1589
1590 if (intel_uc_uses_guc_submission(&i915->gt.uc))
1591 return -ENODEV; /* not implement yet */
1592
1593 if (get_user(idx, &ext->engine_index))
1594 return -EFAULT;
1595
1596 if (idx >= set->engines->num_engines) {
1597 drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n",
1598 idx, set->engines->num_engines);
1599 return -EINVAL;
1600 }
1601
1602 idx = array_index_nospec(idx, set->engines->num_engines);
1603 if (set->engines->engines[idx]) {
1604 drm_dbg(&i915->drm,
1605 "Invalid placement[%d], already occupied\n", idx);
1606 return -EEXIST;
1607 }
1608
1609 if (get_user(num_siblings, &ext->num_siblings))
1610 return -EFAULT;
1611
1612 err = check_user_mbz(&ext->flags);
1613 if (err)
1614 return err;
1615
1616 err = check_user_mbz(&ext->mbz64);
1617 if (err)
1618 return err;
1619
1620 siblings = stack;
1621 if (num_siblings > ARRAY_SIZE(stack)) {
1622 siblings = kmalloc_array(num_siblings,
1623 sizeof(*siblings),
1624 GFP_KERNEL);
1625 if (!siblings)
1626 return -ENOMEM;
1627 }
1628
1629 for (n = 0; n < num_siblings; n++) {
1630 struct i915_engine_class_instance ci;
1631
1632 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) {
1633 err = -EFAULT;
1634 goto out_siblings;
1635 }
1636
1637 siblings[n] = intel_engine_lookup_user(i915,
1638 ci.engine_class,
1639 ci.engine_instance);
1640 if (!siblings[n]) {
1641 drm_dbg(&i915->drm,
1642 "Invalid sibling[%d]: { class:%d, inst:%d }\n",
1643 n, ci.engine_class, ci.engine_instance);
1644 err = -EINVAL;
1645 goto out_siblings;
1646 }
1647 }
1648
1649 ce = intel_execlists_create_virtual(siblings, n);
1650 if (IS_ERR(ce)) {
1651 err = PTR_ERR(ce);
1652 goto out_siblings;
1653 }
1654
1655 intel_context_set_gem(ce, set->ctx);
1656
1657 if (cmpxchg(&set->engines->engines[idx], NULL, ce)) {
1658 intel_context_put(ce);
1659 err = -EEXIST;
1660 goto out_siblings;
1661 }
1662
1663out_siblings:
1664 if (siblings != stack)
1665 kfree(siblings);
1666
1667 return err;
1668}
1669
1670static int
1671set_engines__bond(struct i915_user_extension __user *base, void *data)
1672{
1673 struct i915_context_engines_bond __user *ext =
1674 container_of_user(base, typeof(*ext), base);
1675 const struct set_engines *set = data;
1676 struct drm_i915_private *i915 = set->ctx->i915;
1677 struct i915_engine_class_instance ci;
1678 struct intel_engine_cs *virtual;
1679 struct intel_engine_cs *master;
1680 u16 idx, num_bonds;
1681 int err, n;
1682
1683 if (get_user(idx, &ext->virtual_index))
1684 return -EFAULT;
1685
1686 if (idx >= set->engines->num_engines) {
1687 drm_dbg(&i915->drm,
1688 "Invalid index for virtual engine: %d >= %d\n",
1689 idx, set->engines->num_engines);
1690 return -EINVAL;
1691 }
1692
1693 idx = array_index_nospec(idx, set->engines->num_engines);
1694 if (!set->engines->engines[idx]) {
1695 drm_dbg(&i915->drm, "Invalid engine at %d\n", idx);
1696 return -EINVAL;
1697 }
1698 virtual = set->engines->engines[idx]->engine;
1699
1700 err = check_user_mbz(&ext->flags);
1701 if (err)
1702 return err;
1703
1704 for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) {
1705 err = check_user_mbz(&ext->mbz64[n]);
1706 if (err)
1707 return err;
1708 }
1709
1710 if (copy_from_user(&ci, &ext->master, sizeof(ci)))
1711 return -EFAULT;
1712
1713 master = intel_engine_lookup_user(i915,
1714 ci.engine_class, ci.engine_instance);
1715 if (!master) {
1716 drm_dbg(&i915->drm,
1717 "Unrecognised master engine: { class:%u, instance:%u }\n",
1718 ci.engine_class, ci.engine_instance);
1719 return -EINVAL;
1720 }
1721
1722 if (get_user(num_bonds, &ext->num_bonds))
1723 return -EFAULT;
1724
1725 for (n = 0; n < num_bonds; n++) {
1726 struct intel_engine_cs *bond;
1727
1728 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci)))
1729 return -EFAULT;
1730
1731 bond = intel_engine_lookup_user(i915,
1732 ci.engine_class,
1733 ci.engine_instance);
1734 if (!bond) {
1735 drm_dbg(&i915->drm,
1736 "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n",
1737 n, ci.engine_class, ci.engine_instance);
1738 return -EINVAL;
1739 }
1740
1741 /*
1742 * A non-virtual engine has no siblings to choose between; and
1743 * a submit fence will always be directed to the one engine.
1744 */
1745 if (intel_engine_is_virtual(virtual)) {
1746 err = intel_virtual_engine_attach_bond(virtual,
1747 master,
1748 bond);
1749 if (err)
1750 return err;
1751 }
1752 }
1753
1754 return 0;
1755}
1756
1757static const i915_user_extension_fn set_engines__extensions[] = {
1758 [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance,
1759 [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond,
1760};
1761
1762static int
1763set_engines(struct i915_gem_context *ctx,
1764 const struct drm_i915_gem_context_param *args)
1765{
1766 struct drm_i915_private *i915 = ctx->i915;
1767 struct i915_context_param_engines __user *user =
1768 u64_to_user_ptr(args->value);
1769 struct set_engines set = { .ctx = ctx };
1770 unsigned int num_engines, n;
1771 u64 extensions;
1772 int err;
1773
1774 if (!args->size) { /* switch back to legacy user_ring_map */
1775 if (!i915_gem_context_user_engines(ctx))
1776 return 0;
1777
1778 set.engines = default_engines(ctx);
1779 if (IS_ERR(set.engines))
1780 return PTR_ERR(set.engines);
1781
1782 goto replace;
1783 }
1784
1785 BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines)));
1786 if (args->size < sizeof(*user) ||
1787 !IS_ALIGNED(args->size, sizeof(*user->engines))) {
1788 drm_dbg(&i915->drm, "Invalid size for engine array: %d\n",
1789 args->size);
1790 return -EINVAL;
1791 }
1792
1793 /*
1794 * Note that I915_EXEC_RING_MASK limits execbuf to only using the
1795 * first 64 engines defined here.
1796 */
1797 num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines);
1798 set.engines = alloc_engines(num_engines);
1799 if (!set.engines)
1800 return -ENOMEM;
1801
1802 for (n = 0; n < num_engines; n++) {
1803 struct i915_engine_class_instance ci;
1804 struct intel_engine_cs *engine;
1805 struct intel_context *ce;
1806
1807 if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) {
1808 __free_engines(set.engines, n);
1809 return -EFAULT;
1810 }
1811
1812 if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID &&
1813 ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) {
1814 set.engines->engines[n] = NULL;
1815 continue;
1816 }
1817
1818 engine = intel_engine_lookup_user(ctx->i915,
1819 ci.engine_class,
1820 ci.engine_instance);
1821 if (!engine) {
1822 drm_dbg(&i915->drm,
1823 "Invalid engine[%d]: { class:%d, instance:%d }\n",
1824 n, ci.engine_class, ci.engine_instance);
1825 __free_engines(set.engines, n);
1826 return -ENOENT;
1827 }
1828
1829 ce = intel_context_create(engine);
1830 if (IS_ERR(ce)) {
1831 __free_engines(set.engines, n);
1832 return PTR_ERR(ce);
1833 }
1834
1835 intel_context_set_gem(ce, ctx);
1836
1837 set.engines->engines[n] = ce;
1838 }
1839 set.engines->num_engines = num_engines;
1840
1841 err = -EFAULT;
1842 if (!get_user(extensions, &user->extensions))
1843 err = i915_user_extensions(u64_to_user_ptr(extensions),
1844 set_engines__extensions,
1845 ARRAY_SIZE(set_engines__extensions),
1846 &set);
1847 if (err) {
1848 free_engines(set.engines);
1849 return err;
1850 }
1851
1852replace:
1853 mutex_lock(&ctx->engines_mutex);
1854 if (i915_gem_context_is_closed(ctx)) {
1855 mutex_unlock(&ctx->engines_mutex);
1856 free_engines(set.engines);
1857 return -ENOENT;
1858 }
1859 if (args->size)
1860 i915_gem_context_set_user_engines(ctx);
1861 else
1862 i915_gem_context_clear_user_engines(ctx);
1863 set.engines = rcu_replace_pointer(ctx->engines, set.engines, 1);
1864 mutex_unlock(&ctx->engines_mutex);
1865
1866 /* Keep track of old engine sets for kill_context() */
1867 engines_idle_release(ctx, set.engines);
1868
1869 return 0;
1870}
1871
1872static struct i915_gem_engines *
1873__copy_engines(struct i915_gem_engines *e)
1874{
1875 struct i915_gem_engines *copy;
1876 unsigned int n;
1877
1878 copy = alloc_engines(e->num_engines);
1879 if (!copy)
1880 return ERR_PTR(-ENOMEM);
1881
1882 for (n = 0; n < e->num_engines; n++) {
1883 if (e->engines[n])
1884 copy->engines[n] = intel_context_get(e->engines[n]);
1885 else
1886 copy->engines[n] = NULL;
1887 }
1888 copy->num_engines = n;
1889
1890 return copy;
1891}
1892
1893static int
1894get_engines(struct i915_gem_context *ctx,
1895 struct drm_i915_gem_context_param *args)
1896{
1897 struct i915_context_param_engines __user *user;
1898 struct i915_gem_engines *e;
1899 size_t n, count, size;
1900 int err = 0;
1901
1902 err = mutex_lock_interruptible(&ctx->engines_mutex);
1903 if (err)
1904 return err;
1905
1906 e = NULL;
1907 if (i915_gem_context_user_engines(ctx))
1908 e = __copy_engines(i915_gem_context_engines(ctx));
1909 mutex_unlock(&ctx->engines_mutex);
1910 if (IS_ERR_OR_NULL(e)) {
1911 args->size = 0;
1912 return PTR_ERR_OR_ZERO(e);
1913 }
1914
1915 count = e->num_engines;
1916
1917 /* Be paranoid in case we have an impedance mismatch */
1918 if (!check_struct_size(user, engines, count, &size)) {
1919 err = -EINVAL;
1920 goto err_free;
1921 }
1922 if (overflows_type(size, args->size)) {
1923 err = -EINVAL;
1924 goto err_free;
1925 }
1926
1927 if (!args->size) {
1928 args->size = size;
1929 goto err_free;
1930 }
1931
1932 if (args->size < size) {
1933 err = -EINVAL;
1934 goto err_free;
1935 }
1936
1937 user = u64_to_user_ptr(args->value);
1938 if (put_user(0, &user->extensions)) {
1939 err = -EFAULT;
1940 goto err_free;
1941 }
1942
1943 for (n = 0; n < count; n++) {
1944 struct i915_engine_class_instance ci = {
1945 .engine_class = I915_ENGINE_CLASS_INVALID,
1946 .engine_instance = I915_ENGINE_CLASS_INVALID_NONE,
1947 };
1948
1949 if (e->engines[n]) {
1950 ci.engine_class = e->engines[n]->engine->uabi_class;
1951 ci.engine_instance = e->engines[n]->engine->uabi_instance;
1952 }
1953
1954 if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) {
1955 err = -EFAULT;
1956 goto err_free;
1957 }
1958 }
1959
1960 args->size = size;
1961
1962err_free:
1963 free_engines(e);
1964 return err;
1965}
1966
1967static int
1968set_persistence(struct i915_gem_context *ctx,
1969 const struct drm_i915_gem_context_param *args)
1970{
1971 if (args->size)
1972 return -EINVAL;
1973
1974 return __context_set_persistence(ctx, args->value);
1975}
1976
1977static int __apply_priority(struct intel_context *ce, void *arg)
1978{
1979 struct i915_gem_context *ctx = arg;
1980
1981 if (!intel_engine_has_timeslices(ce->engine))
1982 return 0;
1983
1984 if (ctx->sched.priority >= I915_PRIORITY_NORMAL)
1985 intel_context_set_use_semaphores(ce);
1986 else
1987 intel_context_clear_use_semaphores(ce);
1988
1989 return 0;
1990}
1991
1992static int set_priority(struct i915_gem_context *ctx,
1993 const struct drm_i915_gem_context_param *args)
1994{
1995 s64 priority = args->value;
1996
1997 if (args->size)
1998 return -EINVAL;
1999
2000 if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
2001 return -ENODEV;
2002
2003 if (priority > I915_CONTEXT_MAX_USER_PRIORITY ||
2004 priority < I915_CONTEXT_MIN_USER_PRIORITY)
2005 return -EINVAL;
2006
2007 if (priority > I915_CONTEXT_DEFAULT_PRIORITY &&
2008 !capable(CAP_SYS_NICE))
2009 return -EPERM;
2010
2011 ctx->sched.priority = I915_USER_PRIORITY(priority);
2012 context_apply_all(ctx, __apply_priority, ctx);
2013
2014 return 0;
2015}
2016
2017static int ctx_setparam(struct drm_i915_file_private *fpriv,
2018 struct i915_gem_context *ctx,
2019 struct drm_i915_gem_context_param *args)
2020{
2021 int ret = 0;
2022
2023 switch (args->param) {
2024 case I915_CONTEXT_PARAM_NO_ZEROMAP:
2025 if (args->size)
2026 ret = -EINVAL;
2027 else if (args->value)
2028 set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags);
2029 else
2030 clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags);
2031 break;
2032
2033 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
2034 if (args->size)
2035 ret = -EINVAL;
2036 else if (args->value)
2037 i915_gem_context_set_no_error_capture(ctx);
2038 else
2039 i915_gem_context_clear_no_error_capture(ctx);
2040 break;
2041
2042 case I915_CONTEXT_PARAM_BANNABLE:
2043 if (args->size)
2044 ret = -EINVAL;
2045 else if (!capable(CAP_SYS_ADMIN) && !args->value)
2046 ret = -EPERM;
2047 else if (args->value)
2048 i915_gem_context_set_bannable(ctx);
2049 else
2050 i915_gem_context_clear_bannable(ctx);
2051 break;
2052
2053 case I915_CONTEXT_PARAM_RECOVERABLE:
2054 if (args->size)
2055 ret = -EINVAL;
2056 else if (args->value)
2057 i915_gem_context_set_recoverable(ctx);
2058 else
2059 i915_gem_context_clear_recoverable(ctx);
2060 break;
2061
2062 case I915_CONTEXT_PARAM_PRIORITY:
2063 ret = set_priority(ctx, args);
2064 break;
2065
2066 case I915_CONTEXT_PARAM_SSEU:
2067 ret = set_sseu(ctx, args);
2068 break;
2069
2070 case I915_CONTEXT_PARAM_VM:
2071 ret = set_ppgtt(fpriv, ctx, args);
2072 break;
2073
2074 case I915_CONTEXT_PARAM_ENGINES:
2075 ret = set_engines(ctx, args);
2076 break;
2077
2078 case I915_CONTEXT_PARAM_PERSISTENCE:
2079 ret = set_persistence(ctx, args);
2080 break;
2081
2082 case I915_CONTEXT_PARAM_RINGSIZE:
2083 ret = set_ringsize(ctx, args);
2084 break;
2085
2086 case I915_CONTEXT_PARAM_BAN_PERIOD:
2087 default:
2088 ret = -EINVAL;
2089 break;
2090 }
2091
2092 return ret;
2093}
2094
2095struct create_ext {
2096 struct i915_gem_context *ctx;
2097 struct drm_i915_file_private *fpriv;
2098};
2099
2100static int create_setparam(struct i915_user_extension __user *ext, void *data)
2101{
2102 struct drm_i915_gem_context_create_ext_setparam local;
2103 const struct create_ext *arg = data;
2104
2105 if (copy_from_user(&local, ext, sizeof(local)))
2106 return -EFAULT;
2107
2108 if (local.param.ctx_id)
2109 return -EINVAL;
2110
2111 return ctx_setparam(arg->fpriv, arg->ctx, &local.param);
2112}
2113
2114static int copy_ring_size(struct intel_context *dst,
2115 struct intel_context *src)
2116{
2117 long sz;
2118
2119 sz = intel_context_get_ring_size(src);
2120 if (sz < 0)
2121 return sz;
2122
2123 return intel_context_set_ring_size(dst, sz);
2124}
2125
2126static int clone_engines(struct i915_gem_context *dst,
2127 struct i915_gem_context *src)
2128{
2129 struct i915_gem_engines *e = i915_gem_context_lock_engines(src);
2130 struct i915_gem_engines *clone;
2131 bool user_engines;
2132 unsigned long n;
2133
2134 clone = alloc_engines(e->num_engines);
2135 if (!clone)
2136 goto err_unlock;
2137
2138 for (n = 0; n < e->num_engines; n++) {
2139 struct intel_engine_cs *engine;
2140
2141 if (!e->engines[n]) {
2142 clone->engines[n] = NULL;
2143 continue;
2144 }
2145 engine = e->engines[n]->engine;
2146
2147 /*
2148 * Virtual engines are singletons; they can only exist
2149 * inside a single context, because they embed their
2150 * HW context... As each virtual context implies a single
2151 * timeline (each engine can only dequeue a single request
2152 * at any time), it would be surprising for two contexts
2153 * to use the same engine. So let's create a copy of
2154 * the virtual engine instead.
2155 */
2156 if (intel_engine_is_virtual(engine))
2157 clone->engines[n] =
2158 intel_execlists_clone_virtual(engine);
2159 else
2160 clone->engines[n] = intel_context_create(engine);
2161 if (IS_ERR_OR_NULL(clone->engines[n])) {
2162 __free_engines(clone, n);
2163 goto err_unlock;
2164 }
2165
2166 intel_context_set_gem(clone->engines[n], dst);
2167
2168 /* Copy across the preferred ringsize */
2169 if (copy_ring_size(clone->engines[n], e->engines[n])) {
2170 __free_engines(clone, n + 1);
2171 goto err_unlock;
2172 }
2173 }
2174 clone->num_engines = n;
2175
2176 user_engines = i915_gem_context_user_engines(src);
2177 i915_gem_context_unlock_engines(src);
2178
2179 /* Serialised by constructor */
2180 engines_idle_release(dst, rcu_replace_pointer(dst->engines, clone, 1));
2181 if (user_engines)
2182 i915_gem_context_set_user_engines(dst);
2183 else
2184 i915_gem_context_clear_user_engines(dst);
2185 return 0;
2186
2187err_unlock:
2188 i915_gem_context_unlock_engines(src);
2189 return -ENOMEM;
2190}
2191
2192static int clone_flags(struct i915_gem_context *dst,
2193 struct i915_gem_context *src)
2194{
2195 dst->user_flags = src->user_flags;
2196 return 0;
2197}
2198
2199static int clone_schedattr(struct i915_gem_context *dst,
2200 struct i915_gem_context *src)
2201{
2202 dst->sched = src->sched;
2203 return 0;
2204}
2205
2206static int clone_sseu(struct i915_gem_context *dst,
2207 struct i915_gem_context *src)
2208{
2209 struct i915_gem_engines *e = i915_gem_context_lock_engines(src);
2210 struct i915_gem_engines *clone;
2211 unsigned long n;
2212 int err;
2213
2214 /* no locking required; sole access under constructor*/
2215 clone = __context_engines_static(dst);
2216 if (e->num_engines != clone->num_engines) {
2217 err = -EINVAL;
2218 goto unlock;
2219 }
2220
2221 for (n = 0; n < e->num_engines; n++) {
2222 struct intel_context *ce = e->engines[n];
2223
2224 if (clone->engines[n]->engine->class != ce->engine->class) {
2225 /* Must have compatible engine maps! */
2226 err = -EINVAL;
2227 goto unlock;
2228 }
2229
2230 /* serialises with set_sseu */
2231 err = intel_context_lock_pinned(ce);
2232 if (err)
2233 goto unlock;
2234
2235 clone->engines[n]->sseu = ce->sseu;
2236 intel_context_unlock_pinned(ce);
2237 }
2238
2239 err = 0;
2240unlock:
2241 i915_gem_context_unlock_engines(src);
2242 return err;
2243}
2244
2245static int clone_timeline(struct i915_gem_context *dst,
2246 struct i915_gem_context *src)
2247{
2248 if (src->timeline)
2249 __assign_timeline(dst, src->timeline);
2250
2251 return 0;
2252}
2253
2254static int clone_vm(struct i915_gem_context *dst,
2255 struct i915_gem_context *src)
2256{
2257 struct i915_address_space *vm;
2258 int err = 0;
2259
2260 if (!rcu_access_pointer(src->vm))
2261 return 0;
2262
2263 rcu_read_lock();
2264 vm = context_get_vm_rcu(src);
2265 rcu_read_unlock();
2266
2267 if (!mutex_lock_interruptible(&dst->mutex)) {
2268 __assign_ppgtt(dst, vm);
2269 mutex_unlock(&dst->mutex);
2270 } else {
2271 err = -EINTR;
2272 }
2273
2274 i915_vm_put(vm);
2275 return err;
2276}
2277
2278static int create_clone(struct i915_user_extension __user *ext, void *data)
2279{
2280 static int (* const fn[])(struct i915_gem_context *dst,
2281 struct i915_gem_context *src) = {
2282#define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y
2283 MAP(ENGINES, clone_engines),
2284 MAP(FLAGS, clone_flags),
2285 MAP(SCHEDATTR, clone_schedattr),
2286 MAP(SSEU, clone_sseu),
2287 MAP(TIMELINE, clone_timeline),
2288 MAP(VM, clone_vm),
2289#undef MAP
2290 };
2291 struct drm_i915_gem_context_create_ext_clone local;
2292 const struct create_ext *arg = data;
2293 struct i915_gem_context *dst = arg->ctx;
2294 struct i915_gem_context *src;
2295 int err, bit;
2296
2297 if (copy_from_user(&local, ext, sizeof(local)))
2298 return -EFAULT;
2299
2300 BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) !=
2301 I915_CONTEXT_CLONE_UNKNOWN);
2302
2303 if (local.flags & I915_CONTEXT_CLONE_UNKNOWN)
2304 return -EINVAL;
2305
2306 if (local.rsvd)
2307 return -EINVAL;
2308
2309 rcu_read_lock();
2310 src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id);
2311 rcu_read_unlock();
2312 if (!src)
2313 return -ENOENT;
2314
2315 GEM_BUG_ON(src == dst);
2316
2317 for (bit = 0; bit < ARRAY_SIZE(fn); bit++) {
2318 if (!(local.flags & BIT(bit)))
2319 continue;
2320
2321 err = fn[bit](dst, src);
2322 if (err)
2323 return err;
2324 }
2325
2326 return 0;
2327}
2328
2329static const i915_user_extension_fn create_extensions[] = {
2330 [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam,
2331 [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone,
2332};
2333
2334static bool client_is_banned(struct drm_i915_file_private *file_priv)
2335{
2336 return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED;
2337}
2338
2339int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
2340 struct drm_file *file)
2341{
2342 struct drm_i915_private *i915 = to_i915(dev);
2343 struct drm_i915_gem_context_create_ext *args = data;
2344 struct create_ext ext_data;
2345 int ret;
2346 u32 id;
2347
2348 if (!DRIVER_CAPS(i915)->has_logical_contexts)
2349 return -ENODEV;
2350
2351 if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN)
2352 return -EINVAL;
2353
2354 ret = intel_gt_terminally_wedged(&i915->gt);
2355 if (ret)
2356 return ret;
2357
2358 ext_data.fpriv = file->driver_priv;
2359 if (client_is_banned(ext_data.fpriv)) {
2360 drm_dbg(&i915->drm,
2361 "client %s[%d] banned from creating ctx\n",
2362 current->comm, task_pid_nr(current));
2363 return -EIO;
2364 }
2365
2366 ext_data.ctx = i915_gem_create_context(i915, args->flags);
2367 if (IS_ERR(ext_data.ctx))
2368 return PTR_ERR(ext_data.ctx);
2369
2370 if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) {
2371 ret = i915_user_extensions(u64_to_user_ptr(args->extensions),
2372 create_extensions,
2373 ARRAY_SIZE(create_extensions),
2374 &ext_data);
2375 if (ret)
2376 goto err_ctx;
2377 }
2378
2379 ret = gem_context_register(ext_data.ctx, ext_data.fpriv, &id);
2380 if (ret < 0)
2381 goto err_ctx;
2382
2383 args->ctx_id = id;
2384 drm_dbg(&i915->drm, "HW context %d created\n", args->ctx_id);
2385
2386 return 0;
2387
2388err_ctx:
2389 context_close(ext_data.ctx);
2390 return ret;
2391}
2392
2393int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
2394 struct drm_file *file)
2395{
2396 struct drm_i915_gem_context_destroy *args = data;
2397 struct drm_i915_file_private *file_priv = file->driver_priv;
2398 struct i915_gem_context *ctx;
2399
2400 if (args->pad != 0)
2401 return -EINVAL;
2402
2403 if (!args->ctx_id)
2404 return -ENOENT;
2405
2406 ctx = xa_erase(&file_priv->context_xa, args->ctx_id);
2407 if (!ctx)
2408 return -ENOENT;
2409
2410 context_close(ctx);
2411 return 0;
2412}
2413
2414static int get_sseu(struct i915_gem_context *ctx,
2415 struct drm_i915_gem_context_param *args)
2416{
2417 struct drm_i915_gem_context_param_sseu user_sseu;
2418 struct intel_context *ce;
2419 unsigned long lookup;
2420 int err;
2421
2422 if (args->size == 0)
2423 goto out;
2424 else if (args->size < sizeof(user_sseu))
2425 return -EINVAL;
2426
2427 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
2428 sizeof(user_sseu)))
2429 return -EFAULT;
2430
2431 if (user_sseu.rsvd)
2432 return -EINVAL;
2433
2434 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX))
2435 return -EINVAL;
2436
2437 lookup = 0;
2438 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)
2439 lookup |= LOOKUP_USER_INDEX;
2440
2441 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine);
2442 if (IS_ERR(ce))
2443 return PTR_ERR(ce);
2444
2445 err = intel_context_lock_pinned(ce); /* serialises with set_sseu */
2446 if (err) {
2447 intel_context_put(ce);
2448 return err;
2449 }
2450
2451 user_sseu.slice_mask = ce->sseu.slice_mask;
2452 user_sseu.subslice_mask = ce->sseu.subslice_mask;
2453 user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice;
2454 user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice;
2455
2456 intel_context_unlock_pinned(ce);
2457 intel_context_put(ce);
2458
2459 if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu,
2460 sizeof(user_sseu)))
2461 return -EFAULT;
2462
2463out:
2464 args->size = sizeof(user_sseu);
2465
2466 return 0;
2467}
2468
2469int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
2470 struct drm_file *file)
2471{
2472 struct drm_i915_file_private *file_priv = file->driver_priv;
2473 struct drm_i915_gem_context_param *args = data;
2474 struct i915_gem_context *ctx;
2475 int ret = 0;
2476
2477 ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
2478 if (!ctx)
2479 return -ENOENT;
2480
2481 switch (args->param) {
2482 case I915_CONTEXT_PARAM_NO_ZEROMAP:
2483 args->size = 0;
2484 args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags);
2485 break;
2486
2487 case I915_CONTEXT_PARAM_GTT_SIZE:
2488 args->size = 0;
2489 rcu_read_lock();
2490 if (rcu_access_pointer(ctx->vm))
2491 args->value = rcu_dereference(ctx->vm)->total;
2492 else
2493 args->value = to_i915(dev)->ggtt.vm.total;
2494 rcu_read_unlock();
2495 break;
2496
2497 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
2498 args->size = 0;
2499 args->value = i915_gem_context_no_error_capture(ctx);
2500 break;
2501
2502 case I915_CONTEXT_PARAM_BANNABLE:
2503 args->size = 0;
2504 args->value = i915_gem_context_is_bannable(ctx);
2505 break;
2506
2507 case I915_CONTEXT_PARAM_RECOVERABLE:
2508 args->size = 0;
2509 args->value = i915_gem_context_is_recoverable(ctx);
2510 break;
2511
2512 case I915_CONTEXT_PARAM_PRIORITY:
2513 args->size = 0;
2514 args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT;
2515 break;
2516
2517 case I915_CONTEXT_PARAM_SSEU:
2518 ret = get_sseu(ctx, args);
2519 break;
2520
2521 case I915_CONTEXT_PARAM_VM:
2522 ret = get_ppgtt(file_priv, ctx, args);
2523 break;
2524
2525 case I915_CONTEXT_PARAM_ENGINES:
2526 ret = get_engines(ctx, args);
2527 break;
2528
2529 case I915_CONTEXT_PARAM_PERSISTENCE:
2530 args->size = 0;
2531 args->value = i915_gem_context_is_persistent(ctx);
2532 break;
2533
2534 case I915_CONTEXT_PARAM_RINGSIZE:
2535 ret = get_ringsize(ctx, args);
2536 break;
2537
2538 case I915_CONTEXT_PARAM_BAN_PERIOD:
2539 default:
2540 ret = -EINVAL;
2541 break;
2542 }
2543
2544 i915_gem_context_put(ctx);
2545 return ret;
2546}
2547
2548int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
2549 struct drm_file *file)
2550{
2551 struct drm_i915_file_private *file_priv = file->driver_priv;
2552 struct drm_i915_gem_context_param *args = data;
2553 struct i915_gem_context *ctx;
2554 int ret;
2555
2556 ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
2557 if (!ctx)
2558 return -ENOENT;
2559
2560 ret = ctx_setparam(file_priv, ctx, args);
2561
2562 i915_gem_context_put(ctx);
2563 return ret;
2564}
2565
2566int i915_gem_context_reset_stats_ioctl(struct drm_device *dev,
2567 void *data, struct drm_file *file)
2568{
2569 struct drm_i915_private *i915 = to_i915(dev);
2570 struct drm_i915_reset_stats *args = data;
2571 struct i915_gem_context *ctx;
2572 int ret;
2573
2574 if (args->flags || args->pad)
2575 return -EINVAL;
2576
2577 ret = -ENOENT;
2578 rcu_read_lock();
2579 ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id);
2580 if (!ctx)
2581 goto out;
2582
2583 /*
2584 * We opt for unserialised reads here. This may result in tearing
2585 * in the extremely unlikely event of a GPU hang on this context
2586 * as we are querying them. If we need that extra layer of protection,
2587 * we should wrap the hangstats with a seqlock.
2588 */
2589
2590 if (capable(CAP_SYS_ADMIN))
2591 args->reset_count = i915_reset_count(&i915->gpu_error);
2592 else
2593 args->reset_count = 0;
2594
2595 args->batch_active = atomic_read(&ctx->guilty_count);
2596 args->batch_pending = atomic_read(&ctx->active_count);
2597
2598 ret = 0;
2599out:
2600 rcu_read_unlock();
2601 return ret;
2602}
2603
2604/* GEM context-engines iterator: for_each_gem_engine() */
2605struct intel_context *
2606i915_gem_engines_iter_next(struct i915_gem_engines_iter *it)
2607{
2608 const struct i915_gem_engines *e = it->engines;
2609 struct intel_context *ctx;
2610
2611 if (unlikely(!e))
2612 return NULL;
2613
2614 do {
2615 if (it->idx >= e->num_engines)
2616 return NULL;
2617
2618 ctx = e->engines[it->idx++];
2619 } while (!ctx);
2620
2621 return ctx;
2622}
2623
2624#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2625#include "selftests/mock_context.c"
2626#include "selftests/i915_gem_context.c"
2627#endif
2628
2629static void i915_global_gem_context_shrink(void)
2630{
2631 kmem_cache_shrink(global.slab_luts);
2632}
2633
2634static void i915_global_gem_context_exit(void)
2635{
2636 kmem_cache_destroy(global.slab_luts);
2637}
2638
2639static struct i915_global_gem_context global = { {
2640 .shrink = i915_global_gem_context_shrink,
2641 .exit = i915_global_gem_context_exit,
2642} };
2643
2644int __init i915_global_gem_context_init(void)
2645{
2646 global.slab_luts = KMEM_CACHE(i915_lut_handle, 0);
2647 if (!global.slab_luts)
2648 return -ENOMEM;
2649
2650 i915_global_register(&global.base);
2651 return 0;
2652}