1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include <linux/workqueue.h>
  7
  8#include "i915_drv.h" /* for_each_engine() */
  9#include "i915_request.h"
 10#include "intel_engine_heartbeat.h"
 11#include "intel_execlists_submission.h"
 12#include "intel_gt.h"
 13#include "intel_gt_pm.h"
 14#include "intel_gt_requests.h"
 15#include "intel_timeline.h"
 16
 17static bool retire_requests(struct intel_timeline *tl)
 18{
 19	struct i915_request *rq, *rn;
 20
 21	list_for_each_entry_safe(rq, rn, &tl->requests, link)
 22		if (!i915_request_retire(rq))
 23			return false;
 24
 25	/* And check nothing new was submitted */
 26	return !i915_active_fence_isset(&tl->last_request);
 27}
 28
 29static bool engine_active(const struct intel_engine_cs *engine)
 30{
 31	return !list_empty(&engine->kernel_context->timeline->requests);
 32}
 33
 34static bool flush_submission(struct intel_gt *gt, long timeout)
 35{
 36	struct intel_engine_cs *engine;
 37	enum intel_engine_id id;
 38	bool active = false;
 39
 40	if (!timeout)
 41		return false;
 42
 43	if (!intel_gt_pm_is_awake(gt))
 44		return false;
 45
 46	for_each_engine(engine, gt, id) {
 47		intel_engine_flush_submission(engine);
 48
 49		/* Flush the background retirement and idle barriers */
 50		flush_work(&engine->retire_work);
 51		flush_delayed_work(&engine->wakeref.work);
 52
 53		/* Is the idle barrier still outstanding? */
 54		active |= engine_active(engine);
 55	}
 56
 57	return active;
 58}
 59
 60static void engine_retire(struct work_struct *work)
 61{
 62	struct intel_engine_cs *engine =
 63		container_of(work, typeof(*engine), retire_work);
 64	struct intel_timeline *tl = xchg(&engine->retire, NULL);
 65
 66	do {
 67		struct intel_timeline *next = xchg(&tl->retire, NULL);
 68
 69		/*
 70		 * Our goal here is to retire _idle_ timelines as soon as
 71		 * possible (as they are idle, we do not expect userspace
 72		 * to be cleaning up anytime soon).
 73		 *
 74		 * If the timeline is currently locked, either it is being
 75		 * retired elsewhere or about to be!
 76		 */
 77		if (mutex_trylock(&tl->mutex)) {
 78			retire_requests(tl);
 79			mutex_unlock(&tl->mutex);
 80		}
 81		intel_timeline_put(tl);
 82
 83		GEM_BUG_ON(!next);
 84		tl = ptr_mask_bits(next, 1);
 85	} while (tl);
 86}
 87
 88static bool add_retire(struct intel_engine_cs *engine,
 89		       struct intel_timeline *tl)
 90{
 91#define STUB ((struct intel_timeline *)1)
 92	struct intel_timeline *first;
 93
 94	/*
 95	 * We open-code a llist here to include the additional tag [BIT(0)]
 96	 * so that we know when the timeline is already on a
 97	 * retirement queue: either this engine or another.
 98	 */
 99
100	if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
101		return false;
102
103	intel_timeline_get(tl);
104	first = READ_ONCE(engine->retire);
105	do
106		tl->retire = ptr_pack_bits(first, 1, 1);
107	while (!try_cmpxchg(&engine->retire, &first, tl));
108
109	return !first;
110}
111
112void intel_engine_add_retire(struct intel_engine_cs *engine,
113			     struct intel_timeline *tl)
114{
115	/* We don't deal well with the engine disappearing beneath us */
116	GEM_BUG_ON(intel_engine_is_virtual(engine));
117
118	if (add_retire(engine, tl))
119		schedule_work(&engine->retire_work);
120}
121
122void intel_engine_init_retire(struct intel_engine_cs *engine)
123{
124	INIT_WORK(&engine->retire_work, engine_retire);
125}
126
127void intel_engine_fini_retire(struct intel_engine_cs *engine)
128{
129	flush_work(&engine->retire_work);
130	GEM_BUG_ON(engine->retire);
131}
132
133long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
134{
135	struct intel_gt_timelines *timelines = &gt->timelines;
136	struct intel_timeline *tl, *tn;
137	unsigned long active_count = 0;
138	LIST_HEAD(free);
139
140	flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */
141	spin_lock(&timelines->lock);
142	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
143		if (!mutex_trylock(&tl->mutex)) {
144			active_count++; /* report busy to caller, try again? */
145			continue;
146		}
147
148		intel_timeline_get(tl);
149		GEM_BUG_ON(!atomic_read(&tl->active_count));
150		atomic_inc(&tl->active_count); /* pin the list element */
151		spin_unlock(&timelines->lock);
152
153		if (timeout > 0) {
154			struct dma_fence *fence;
155
156			fence = i915_active_fence_get(&tl->last_request);
157			if (fence) {
158				mutex_unlock(&tl->mutex);
159
160				timeout = dma_fence_wait_timeout(fence,
161								 true,
162								 timeout);
163				dma_fence_put(fence);
164
165				/* Retirement is best effort */
166				if (!mutex_trylock(&tl->mutex)) {
167					active_count++;
168					goto out_active;
169				}
170			}
171		}
172
173		if (!retire_requests(tl))
174			active_count++;
175		mutex_unlock(&tl->mutex);
176
177out_active:	spin_lock(&timelines->lock);
178
179		/* Resume list iteration after reacquiring spinlock */
180		list_safe_reset_next(tl, tn, link);
181		if (atomic_dec_and_test(&tl->active_count))
182			list_del(&tl->link);
183
184		/* Defer the final release to after the spinlock */
185		if (refcount_dec_and_test(&tl->kref.refcount)) {
186			GEM_BUG_ON(atomic_read(&tl->active_count));
187			list_add(&tl->link, &free);
188		}
189	}
190	spin_unlock(&timelines->lock);
191
192	list_for_each_entry_safe(tl, tn, &free, link)
193		__intel_timeline_free(&tl->kref);
194
195	if (flush_submission(gt, timeout)) /* Wait, there's more! */
196		active_count++;
197
198	return active_count ? timeout : 0;
199}
200
201int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
202{
203	/* If the device is asleep, we have no requests outstanding */
204	if (!intel_gt_pm_is_awake(gt))
205		return 0;
206
207	while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
208		cond_resched();
209		if (signal_pending(current))
210			return -EINTR;
211	}
212
213	return timeout;
214}
215
216static void retire_work_handler(struct work_struct *work)
217{
218	struct intel_gt *gt =
219		container_of(work, typeof(*gt), requests.retire_work.work);
220
221	schedule_delayed_work(&gt->requests.retire_work,
222			      round_jiffies_up_relative(HZ));
223	intel_gt_retire_requests(gt);
224}
225
226void intel_gt_init_requests(struct intel_gt *gt)
227{
228	INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
229}
230
231void intel_gt_park_requests(struct intel_gt *gt)
232{
233	cancel_delayed_work(&gt->requests.retire_work);
234}
235
236void intel_gt_unpark_requests(struct intel_gt *gt)
237{
238	schedule_delayed_work(&gt->requests.retire_work,
239			      round_jiffies_up_relative(HZ));
240}
241
242void intel_gt_fini_requests(struct intel_gt *gt)
243{
244	/* Wait until the work is marked as finished before unloading! */
245	cancel_delayed_work_sync(&gt->requests.retire_work);
246
247	flush_work(&gt->watchdog.work);
248}
249
250void intel_gt_watchdog_work(struct work_struct *work)
251{
252	struct intel_gt *gt =
253		container_of(work, typeof(*gt), watchdog.work);
254	struct i915_request *rq, *rn;
255	struct llist_node *first;
256
257	first = llist_del_all(&gt->watchdog.list);
258	if (!first)
259		return;
260
261	llist_for_each_entry_safe(rq, rn, first, watchdog.link) {
262		if (!i915_request_completed(rq)) {
263			struct dma_fence *f = &rq->fence;
264
265			pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",
266				  f->ops->get_driver_name(f),
267				  f->ops->get_timeline_name(f),
268				  f->seqno);
269			i915_request_cancel(rq, -EINTR);
270		}
271		i915_request_put(rq);
272	}
273}