1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include "i915_drv.h"
  7#include "gt/intel_context.h"
  8#include "gt/intel_engine_pm.h"
  9#include "i915_gem_client_blt.h"
 10#include "i915_gem_object_blt.h"
 11
 12struct i915_sleeve {
 13	struct i915_vma *vma;
 14	struct drm_i915_gem_object *obj;
 15	struct sg_table *pages;
 16	struct i915_page_sizes page_sizes;
 17};
 18
 19static int vma_set_pages(struct i915_vma *vma)
 20{
 21	struct i915_sleeve *sleeve = vma->private;
 22
 23	vma->pages = sleeve->pages;
 24	vma->page_sizes = sleeve->page_sizes;
 25
 26	return 0;
 27}
 28
 29static void vma_clear_pages(struct i915_vma *vma)
 30{
 31	GEM_BUG_ON(!vma->pages);
 32	vma->pages = NULL;
 33}
 34
 35static int vma_bind(struct i915_address_space *vm,
 36		    struct i915_vma *vma,
 37		    enum i915_cache_level cache_level,
 38		    u32 flags)
 39{
 40	return vm->vma_ops.bind_vma(vm, vma, cache_level, flags);
 41}
 42
 43static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma)
 44{
 45	vm->vma_ops.unbind_vma(vm, vma);
 46}
 47
 48static const struct i915_vma_ops proxy_vma_ops = {
 49	.set_pages = vma_set_pages,
 50	.clear_pages = vma_clear_pages,
 51	.bind_vma = vma_bind,
 52	.unbind_vma = vma_unbind,
 53};
 54
 55static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
 56					 struct drm_i915_gem_object *obj,
 57					 struct sg_table *pages,
 58					 struct i915_page_sizes *page_sizes)
 59{
 60	struct i915_sleeve *sleeve;
 61	struct i915_vma *vma;
 62	int err;
 63
 64	sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
 65	if (!sleeve)
 66		return ERR_PTR(-ENOMEM);
 67
 68	vma = i915_vma_instance(obj, vm, NULL);
 69	if (IS_ERR(vma)) {
 70		err = PTR_ERR(vma);
 71		goto err_free;
 72	}
 73
 74	vma->private = sleeve;
 75	vma->ops = &proxy_vma_ops;
 76
 77	sleeve->vma = vma;
 78	sleeve->pages = pages;
 79	sleeve->page_sizes = *page_sizes;
 80
 81	return sleeve;
 82
 83err_free:
 84	kfree(sleeve);
 85	return ERR_PTR(err);
 86}
 87
 88static void destroy_sleeve(struct i915_sleeve *sleeve)
 89{
 90	kfree(sleeve);
 91}
 92
 93struct clear_pages_work {
 94	struct dma_fence dma;
 95	struct dma_fence_cb cb;
 96	struct i915_sw_fence wait;
 97	struct work_struct work;
 98	struct irq_work irq_work;
 99	struct i915_sleeve *sleeve;
100	struct intel_context *ce;
101	u32 value;
102};
103
104static const char *clear_pages_work_driver_name(struct dma_fence *fence)
105{
106	return DRIVER_NAME;
107}
108
109static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
110{
111	return "clear";
112}
113
114static void clear_pages_work_release(struct dma_fence *fence)
115{
116	struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
117
118	destroy_sleeve(w->sleeve);
119
120	i915_sw_fence_fini(&w->wait);
121
122	BUILD_BUG_ON(offsetof(typeof(*w), dma));
123	dma_fence_free(&w->dma);
124}
125
126static const struct dma_fence_ops clear_pages_work_ops = {
127	.get_driver_name = clear_pages_work_driver_name,
128	.get_timeline_name = clear_pages_work_timeline_name,
129	.release = clear_pages_work_release,
130};
131
132static void clear_pages_signal_irq_worker(struct irq_work *work)
133{
134	struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
135
136	dma_fence_signal(&w->dma);
137	dma_fence_put(&w->dma);
138}
139
140static void clear_pages_dma_fence_cb(struct dma_fence *fence,
141				     struct dma_fence_cb *cb)
142{
143	struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
144
145	if (fence->error)
146		dma_fence_set_error(&w->dma, fence->error);
147
148	/*
149	 * Push the signalling of the fence into yet another worker to avoid
150	 * the nightmare locking around the fence spinlock.
151	 */
152	irq_work_queue(&w->irq_work);
153}
154
155static void clear_pages_worker(struct work_struct *work)
156{
157	struct clear_pages_work *w = container_of(work, typeof(*w), work);
158	struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
159	struct i915_vma *vma = w->sleeve->vma;
160	struct i915_request *rq;
161	struct i915_vma *batch;
162	int err = w->dma.error;
163
164	if (unlikely(err))
165		goto out_signal;
166
167	if (obj->cache_dirty) {
168		if (i915_gem_object_has_struct_page(obj))
169			drm_clflush_sg(w->sleeve->pages);
170		obj->cache_dirty = false;
171	}
172	obj->read_domains = I915_GEM_GPU_DOMAINS;
173	obj->write_domain = 0;
174
175	err = i915_vma_pin(vma, 0, 0, PIN_USER);
176	if (unlikely(err))
177		goto out_signal;
178
179	batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
180	if (IS_ERR(batch)) {
181		err = PTR_ERR(batch);
182		goto out_unpin;
183	}
184
185	rq = intel_context_create_request(w->ce);
186	if (IS_ERR(rq)) {
187		err = PTR_ERR(rq);
188		goto out_batch;
189	}
190
191	/* There's no way the fence has signalled */
192	if (dma_fence_add_callback(&rq->fence, &w->cb,
193				   clear_pages_dma_fence_cb))
194		GEM_BUG_ON(1);
195
196	err = intel_emit_vma_mark_active(batch, rq);
197	if (unlikely(err))
198		goto out_request;
199
200	if (w->ce->engine->emit_init_breadcrumb) {
201		err = w->ce->engine->emit_init_breadcrumb(rq);
202		if (unlikely(err))
203			goto out_request;
204	}
205
206	/*
207	 * w->dma is already exported via (vma|obj)->resv we need only
208	 * keep track of the GPU activity within this vma/request, and
209	 * propagate the signal from the request to w->dma.
210	 */
211	err = __i915_vma_move_to_active(vma, rq);
212	if (err)
213		goto out_request;
214
215	err = w->ce->engine->emit_bb_start(rq,
216					   batch->node.start, batch->node.size,
217					   0);
218out_request:
219	if (unlikely(err)) {
220		i915_request_set_error_once(rq, err);
221		err = 0;
222	}
223
224	i915_request_add(rq);
225out_batch:
226	intel_emit_vma_release(w->ce, batch);
227out_unpin:
228	i915_vma_unpin(vma);
229out_signal:
230	if (unlikely(err)) {
231		dma_fence_set_error(&w->dma, err);
232		dma_fence_signal(&w->dma);
233		dma_fence_put(&w->dma);
234	}
235}
236
237static int __i915_sw_fence_call
238clear_pages_work_notify(struct i915_sw_fence *fence,
239			enum i915_sw_fence_notify state)
240{
241	struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
242
243	switch (state) {
244	case FENCE_COMPLETE:
245		schedule_work(&w->work);
246		break;
247
248	case FENCE_FREE:
249		dma_fence_put(&w->dma);
250		break;
251	}
252
253	return NOTIFY_DONE;
254}
255
256static DEFINE_SPINLOCK(fence_lock);
257
258/* XXX: better name please */
259int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
260				     struct intel_context *ce,
261				     struct sg_table *pages,
262				     struct i915_page_sizes *page_sizes,
263				     u32 value)
264{
265	struct clear_pages_work *work;
266	struct i915_sleeve *sleeve;
267	int err;
268
269	sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
270	if (IS_ERR(sleeve))
271		return PTR_ERR(sleeve);
272
273	work = kmalloc(sizeof(*work), GFP_KERNEL);
274	if (!work) {
275		destroy_sleeve(sleeve);
276		return -ENOMEM;
277	}
278
279	work->value = value;
280	work->sleeve = sleeve;
281	work->ce = ce;
282
283	INIT_WORK(&work->work, clear_pages_worker);
284
285	init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
286
287	dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
288	i915_sw_fence_init(&work->wait, clear_pages_work_notify);
289
290	i915_gem_object_lock(obj);
291	err = i915_sw_fence_await_reservation(&work->wait,
292					      obj->base.resv, NULL, true, 0,
293					      I915_FENCE_GFP);
294	if (err < 0) {
295		dma_fence_set_error(&work->dma, err);
296	} else {
297		dma_resv_add_excl_fence(obj->base.resv, &work->dma);
298		err = 0;
299	}
300	i915_gem_object_unlock(obj);
301
302	dma_fence_get(&work->dma);
303	i915_sw_fence_commit(&work->wait);
304
305	return err;
306}
307
308#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
309#include "selftests/i915_gem_client_blt.c"
310#endif