1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include "i915_drv.h"
  7#include "gt/intel_context.h"
  8#include "gt/intel_engine_pm.h"
  9#include "gt/intel_engine_pool.h"
 10#include "i915_gem_client_blt.h"
 11#include "i915_gem_object_blt.h"
 12
 13struct i915_sleeve {
 14	struct i915_vma *vma;
 15	struct drm_i915_gem_object *obj;
 16	struct sg_table *pages;
 17	struct i915_page_sizes page_sizes;
 18};
 19
 20static int vma_set_pages(struct i915_vma *vma)
 21{
 22	struct i915_sleeve *sleeve = vma->private;
 23
 24	vma->pages = sleeve->pages;
 25	vma->page_sizes = sleeve->page_sizes;
 26
 27	return 0;
 28}
 29
 30static void vma_clear_pages(struct i915_vma *vma)
 31{
 32	GEM_BUG_ON(!vma->pages);
 33	vma->pages = NULL;
 34}
 35
 36static int vma_bind(struct i915_vma *vma,
 37		    enum i915_cache_level cache_level,
 38		    u32 flags)
 39{
 40	return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
 41}
 42
 43static void vma_unbind(struct i915_vma *vma)
 44{
 45	vma->vm->vma_ops.unbind_vma(vma);
 46}
 47
 48static const struct i915_vma_ops proxy_vma_ops = {
 49	.set_pages = vma_set_pages,
 50	.clear_pages = vma_clear_pages,
 51	.bind_vma = vma_bind,
 52	.unbind_vma = vma_unbind,
 53};
 54
 55static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
 56					 struct drm_i915_gem_object *obj,
 57					 struct sg_table *pages,
 58					 struct i915_page_sizes *page_sizes)
 59{
 60	struct i915_sleeve *sleeve;
 61	struct i915_vma *vma;
 62	int err;
 63
 64	sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
 65	if (!sleeve)
 66		return ERR_PTR(-ENOMEM);
 67
 68	vma = i915_vma_instance(obj, vm, NULL);
 69	if (IS_ERR(vma)) {
 70		err = PTR_ERR(vma);
 71		goto err_free;
 72	}
 73
 74	vma->private = sleeve;
 75	vma->ops = &proxy_vma_ops;
 76
 77	sleeve->vma = vma;
 78	sleeve->pages = pages;
 79	sleeve->page_sizes = *page_sizes;
 80
 81	return sleeve;
 82
 83err_free:
 84	kfree(sleeve);
 85	return ERR_PTR(err);
 86}
 87
 88static void destroy_sleeve(struct i915_sleeve *sleeve)
 89{
 90	kfree(sleeve);
 91}
 92
 93struct clear_pages_work {
 94	struct dma_fence dma;
 95	struct dma_fence_cb cb;
 96	struct i915_sw_fence wait;
 97	struct work_struct work;
 98	struct irq_work irq_work;
 99	struct i915_sleeve *sleeve;
100	struct intel_context *ce;
101	u32 value;
102};
103
104static const char *clear_pages_work_driver_name(struct dma_fence *fence)
105{
106	return DRIVER_NAME;
107}
108
109static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
110{
111	return "clear";
112}
113
114static void clear_pages_work_release(struct dma_fence *fence)
115{
116	struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
117
118	destroy_sleeve(w->sleeve);
119
120	i915_sw_fence_fini(&w->wait);
121
122	BUILD_BUG_ON(offsetof(typeof(*w), dma));
123	dma_fence_free(&w->dma);
124}
125
126static const struct dma_fence_ops clear_pages_work_ops = {
127	.get_driver_name = clear_pages_work_driver_name,
128	.get_timeline_name = clear_pages_work_timeline_name,
129	.release = clear_pages_work_release,
130};
131
132static void clear_pages_signal_irq_worker(struct irq_work *work)
133{
134	struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
135
136	dma_fence_signal(&w->dma);
137	dma_fence_put(&w->dma);
138}
139
140static void clear_pages_dma_fence_cb(struct dma_fence *fence,
141				     struct dma_fence_cb *cb)
142{
143	struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
144
145	if (fence->error)
146		dma_fence_set_error(&w->dma, fence->error);
147
148	/*
149	 * Push the signalling of the fence into yet another worker to avoid
150	 * the nightmare locking around the fence spinlock.
151	 */
152	irq_work_queue(&w->irq_work);
153}
154
155static void clear_pages_worker(struct work_struct *work)
156{
157	struct clear_pages_work *w = container_of(work, typeof(*w), work);
158	struct drm_i915_private *i915 = w->ce->engine->i915;
159	struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
160	struct i915_vma *vma = w->sleeve->vma;
161	struct i915_request *rq;
162	struct i915_vma *batch;
163	int err = w->dma.error;
164
165	if (unlikely(err))
166		goto out_signal;
167
168	if (obj->cache_dirty) {
169		if (i915_gem_object_has_struct_page(obj))
170			drm_clflush_sg(w->sleeve->pages);
171		obj->cache_dirty = false;
172	}
173	obj->read_domains = I915_GEM_GPU_DOMAINS;
174	obj->write_domain = 0;
175
176	/* XXX: we need to kill this */
177	mutex_lock(&i915->drm.struct_mutex);
178	err = i915_vma_pin(vma, 0, 0, PIN_USER);
179	if (unlikely(err))
180		goto out_unlock;
181
182	batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
183	if (IS_ERR(batch)) {
184		err = PTR_ERR(batch);
185		goto out_unpin;
186	}
187
188	rq = intel_context_create_request(w->ce);
189	if (IS_ERR(rq)) {
190		err = PTR_ERR(rq);
191		goto out_batch;
192	}
193
194	/* There's no way the fence has signalled */
195	if (dma_fence_add_callback(&rq->fence, &w->cb,
196				   clear_pages_dma_fence_cb))
197		GEM_BUG_ON(1);
198
199	err = intel_emit_vma_mark_active(batch, rq);
200	if (unlikely(err))
201		goto out_request;
202
203	if (w->ce->engine->emit_init_breadcrumb) {
204		err = w->ce->engine->emit_init_breadcrumb(rq);
205		if (unlikely(err))
206			goto out_request;
207	}
208
209	/*
210	 * w->dma is already exported via (vma|obj)->resv we need only
211	 * keep track of the GPU activity within this vma/request, and
212	 * propagate the signal from the request to w->dma.
213	 */
214	err = i915_active_ref(&vma->active, rq->timeline, rq);
215	if (err)
216		goto out_request;
217
218	err = w->ce->engine->emit_bb_start(rq,
219					   batch->node.start, batch->node.size,
220					   0);
221out_request:
222	if (unlikely(err)) {
223		i915_request_skip(rq, err);
224		err = 0;
225	}
226
227	i915_request_add(rq);
228out_batch:
229	intel_emit_vma_release(w->ce, batch);
230out_unpin:
231	i915_vma_unpin(vma);
232out_unlock:
233	mutex_unlock(&i915->drm.struct_mutex);
234out_signal:
235	if (unlikely(err)) {
236		dma_fence_set_error(&w->dma, err);
237		dma_fence_signal(&w->dma);
238		dma_fence_put(&w->dma);
239	}
240}
241
242static int __i915_sw_fence_call
243clear_pages_work_notify(struct i915_sw_fence *fence,
244			enum i915_sw_fence_notify state)
245{
246	struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
247
248	switch (state) {
249	case FENCE_COMPLETE:
250		schedule_work(&w->work);
251		break;
252
253	case FENCE_FREE:
254		dma_fence_put(&w->dma);
255		break;
256	}
257
258	return NOTIFY_DONE;
259}
260
261static DEFINE_SPINLOCK(fence_lock);
262
263/* XXX: better name please */
264int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
265				     struct intel_context *ce,
266				     struct sg_table *pages,
267				     struct i915_page_sizes *page_sizes,
268				     u32 value)
269{
270	struct clear_pages_work *work;
271	struct i915_sleeve *sleeve;
272	int err;
273
274	sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
275	if (IS_ERR(sleeve))
276		return PTR_ERR(sleeve);
277
278	work = kmalloc(sizeof(*work), GFP_KERNEL);
279	if (!work) {
280		destroy_sleeve(sleeve);
281		return -ENOMEM;
282	}
283
284	work->value = value;
285	work->sleeve = sleeve;
286	work->ce = ce;
287
288	INIT_WORK(&work->work, clear_pages_worker);
289
290	init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
291
292	dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
293	i915_sw_fence_init(&work->wait, clear_pages_work_notify);
294
295	i915_gem_object_lock(obj);
296	err = i915_sw_fence_await_reservation(&work->wait,
297					      obj->base.resv, NULL,
298					      true, I915_FENCE_TIMEOUT,
299					      I915_FENCE_GFP);
300	if (err < 0) {
301		dma_fence_set_error(&work->dma, err);
302	} else {
303		dma_resv_add_excl_fence(obj->base.resv, &work->dma);
304		err = 0;
305	}
306	i915_gem_object_unlock(obj);
307
308	dma_fence_get(&work->dma);
309	i915_sw_fence_commit(&work->wait);
310
311	return err;
312}
313
314#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
315#include "selftests/i915_gem_client_blt.c"
316#endif