1/*
  2 * Copyright 2015 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 *
 23 */
 24#include <linux/kthread.h>
 25#include <linux/wait.h>
 26#include <linux/sched.h>
 27#include <drm/drmP.h>
 28#include "gpu_scheduler.h"
 29
 30#define CREATE_TRACE_POINTS
 31#include "gpu_sched_trace.h"
 32
 33static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
 34static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
 35
 36struct kmem_cache *sched_fence_slab;
 37atomic_t sched_fence_slab_ref = ATOMIC_INIT(0);
 38
 39/* Initialize a given run queue struct */
 40static void amd_sched_rq_init(struct amd_sched_rq *rq)
 41{
 42	spin_lock_init(&rq->lock);
 43	INIT_LIST_HEAD(&rq->entities);
 44	rq->current_entity = NULL;
 45}
 46
 47static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
 48				    struct amd_sched_entity *entity)
 49{
 50	if (!list_empty(&entity->list))
 51		return;
 52	spin_lock(&rq->lock);
 53	list_add_tail(&entity->list, &rq->entities);
 54	spin_unlock(&rq->lock);
 55}
 56
 57static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
 58				       struct amd_sched_entity *entity)
 59{
 60	if (list_empty(&entity->list))
 61		return;
 62	spin_lock(&rq->lock);
 63	list_del_init(&entity->list);
 64	if (rq->current_entity == entity)
 65		rq->current_entity = NULL;
 66	spin_unlock(&rq->lock);
 67}
 68
 69/**
 70 * Select an entity which could provide a job to run
 71 *
 72 * @rq		The run queue to check.
 73 *
 74 * Try to find a ready entity, returns NULL if none found.
 75 */
 76static struct amd_sched_entity *
 77amd_sched_rq_select_entity(struct amd_sched_rq *rq)
 78{
 79	struct amd_sched_entity *entity;
 80
 81	spin_lock(&rq->lock);
 82
 83	entity = rq->current_entity;
 84	if (entity) {
 85		list_for_each_entry_continue(entity, &rq->entities, list) {
 86			if (amd_sched_entity_is_ready(entity)) {
 87				rq->current_entity = entity;
 88				spin_unlock(&rq->lock);
 89				return entity;
 90			}
 91		}
 92	}
 93
 94	list_for_each_entry(entity, &rq->entities, list) {
 95
 96		if (amd_sched_entity_is_ready(entity)) {
 97			rq->current_entity = entity;
 98			spin_unlock(&rq->lock);
 99			return entity;
100		}
101
102		if (entity == rq->current_entity)
103			break;
104	}
105
106	spin_unlock(&rq->lock);
107
108	return NULL;
109}
110
111/**
112 * Init a context entity used by scheduler when submit to HW ring.
113 *
114 * @sched	The pointer to the scheduler
115 * @entity	The pointer to a valid amd_sched_entity
116 * @rq		The run queue this entity belongs
117 * @kernel	If this is an entity for the kernel
118 * @jobs	The max number of jobs in the job queue
119 *
120 * return 0 if succeed. negative error code on failure
121*/
122int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
123			  struct amd_sched_entity *entity,
124			  struct amd_sched_rq *rq,
125			  uint32_t jobs)
126{
127	int r;
128
129	if (!(sched && entity && rq))
130		return -EINVAL;
131
132	memset(entity, 0, sizeof(struct amd_sched_entity));
133	INIT_LIST_HEAD(&entity->list);
134	entity->rq = rq;
135	entity->sched = sched;
136
137	spin_lock_init(&entity->queue_lock);
138	r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
139	if (r)
140		return r;
141
142	atomic_set(&entity->fence_seq, 0);
143	entity->fence_context = fence_context_alloc(1);
144
145	return 0;
146}
147
148/**
149 * Query if entity is initialized
150 *
151 * @sched       Pointer to scheduler instance
152 * @entity	The pointer to a valid scheduler entity
153 *
154 * return true if entity is initialized, false otherwise
155*/
156static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
157					    struct amd_sched_entity *entity)
158{
159	return entity->sched == sched &&
160		entity->rq != NULL;
161}
162
163/**
164 * Check if entity is idle
165 *
166 * @entity	The pointer to a valid scheduler entity
167 *
168 * Return true if entity don't has any unscheduled jobs.
169 */
170static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
171{
172	rmb();
173	if (kfifo_is_empty(&entity->job_queue))
174		return true;
175
176	return false;
177}
178
179/**
180 * Check if entity is ready
181 *
182 * @entity	The pointer to a valid scheduler entity
183 *
184 * Return true if entity could provide a job.
185 */
186static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
187{
188	if (kfifo_is_empty(&entity->job_queue))
189		return false;
190
191	if (ACCESS_ONCE(entity->dependency))
192		return false;
193
194	return true;
195}
196
197/**
198 * Destroy a context entity
199 *
200 * @sched       Pointer to scheduler instance
201 * @entity	The pointer to a valid scheduler entity
202 *
203 * Cleanup and free the allocated resources.
204 */
205void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
206			   struct amd_sched_entity *entity)
207{
208	struct amd_sched_rq *rq = entity->rq;
209
210	if (!amd_sched_entity_is_initialized(sched, entity))
211		return;
212
213	/**
214	 * The client will not queue more IBs during this fini, consume existing
215	 * queued IBs
216	*/
217	wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
218
219	amd_sched_rq_remove_entity(rq, entity);
220	kfifo_free(&entity->job_queue);
221}
222
223static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
224{
225	struct amd_sched_entity *entity =
226		container_of(cb, struct amd_sched_entity, cb);
227	entity->dependency = NULL;
228	fence_put(f);
229	amd_sched_wakeup(entity->sched);
230}
231
232static void amd_sched_entity_clear_dep(struct fence *f, struct fence_cb *cb)
233{
234	struct amd_sched_entity *entity =
235		container_of(cb, struct amd_sched_entity, cb);
236	entity->dependency = NULL;
237	fence_put(f);
238}
239
240static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
241{
242	struct amd_gpu_scheduler *sched = entity->sched;
243	struct fence * fence = entity->dependency;
244	struct amd_sched_fence *s_fence;
245
246	if (fence->context == entity->fence_context) {
247		/* We can ignore fences from ourself */
248		fence_put(entity->dependency);
249		return false;
250	}
251
252	s_fence = to_amd_sched_fence(fence);
253	if (s_fence && s_fence->sched == sched) {
254		/* Fence is from the same scheduler */
255		if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &fence->flags)) {
256			/* Ignore it when it is already scheduled */
257			fence_put(entity->dependency);
258			return false;
259		}
260
261		/* Wait for fence to be scheduled */
262		entity->cb.func = amd_sched_entity_clear_dep;
263		list_add_tail(&entity->cb.node, &s_fence->scheduled_cb);
264		return true;
265	}
266
267	if (!fence_add_callback(entity->dependency, &entity->cb,
268				amd_sched_entity_wakeup))
269		return true;
270
271	fence_put(entity->dependency);
272	return false;
273}
274
275static struct amd_sched_job *
276amd_sched_entity_pop_job(struct amd_sched_entity *entity)
277{
278	struct amd_gpu_scheduler *sched = entity->sched;
279	struct amd_sched_job *sched_job;
280
281	if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
282		return NULL;
283
284	while ((entity->dependency = sched->ops->dependency(sched_job)))
285		if (amd_sched_entity_add_dependency_cb(entity))
286			return NULL;
287
288	return sched_job;
289}
290
291/**
292 * Helper to submit a job to the job queue
293 *
294 * @sched_job		The pointer to job required to submit
295 *
296 * Returns true if we could submit the job.
297 */
298static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
299{
300	struct amd_gpu_scheduler *sched = sched_job->sched;
301	struct amd_sched_entity *entity = sched_job->s_entity;
302	bool added, first = false;
303
304	spin_lock(&entity->queue_lock);
305	added = kfifo_in(&entity->job_queue, &sched_job,
306			sizeof(sched_job)) == sizeof(sched_job);
307
308	if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
309		first = true;
310
311	spin_unlock(&entity->queue_lock);
312
313	/* first job wakes up scheduler */
314	if (first) {
315		/* Add the entity to the run queue */
316		amd_sched_rq_add_entity(entity->rq, entity);
317		amd_sched_wakeup(sched);
318	}
319	return added;
320}
321
322/**
323 * Submit a job to the job queue
324 *
325 * @sched_job		The pointer to job required to submit
326 *
327 * Returns 0 for success, negative error code otherwise.
328 */
329void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
330{
331	struct amd_sched_entity *entity = sched_job->s_entity;
332
333	trace_amd_sched_job(sched_job);
334	wait_event(entity->sched->job_scheduled,
335		   amd_sched_entity_in(sched_job));
336}
337
338/**
339 * Return ture if we can push more jobs to the hw.
340 */
341static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
342{
343	return atomic_read(&sched->hw_rq_count) <
344		sched->hw_submission_limit;
345}
346
347/**
348 * Wake up the scheduler when it is ready
349 */
350static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
351{
352	if (amd_sched_ready(sched))
353		wake_up_interruptible(&sched->wake_up_worker);
354}
355
356/**
357 * Select next entity to process
358*/
359static struct amd_sched_entity *
360amd_sched_select_entity(struct amd_gpu_scheduler *sched)
361{
362	struct amd_sched_entity *entity;
363	int i;
364
365	if (!amd_sched_ready(sched))
366		return NULL;
367
368	/* Kernel run queue has higher priority than normal run queue*/
369	for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++) {
370		entity = amd_sched_rq_select_entity(&sched->sched_rq[i]);
371		if (entity)
372			break;
373	}
374
375	return entity;
376}
377
378static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
379{
380	struct amd_sched_fence *s_fence =
381		container_of(cb, struct amd_sched_fence, cb);
382	struct amd_gpu_scheduler *sched = s_fence->sched;
383	unsigned long flags;
384
385	atomic_dec(&sched->hw_rq_count);
386	amd_sched_fence_signal(s_fence);
387	if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
388		cancel_delayed_work(&s_fence->dwork);
389		spin_lock_irqsave(&sched->fence_list_lock, flags);
390		list_del_init(&s_fence->list);
391		spin_unlock_irqrestore(&sched->fence_list_lock, flags);
392	}
393	trace_amd_sched_process_job(s_fence);
394	fence_put(&s_fence->base);
395	wake_up_interruptible(&sched->wake_up_worker);
396}
397
398static void amd_sched_fence_work_func(struct work_struct *work)
399{
400	struct amd_sched_fence *s_fence =
401		container_of(work, struct amd_sched_fence, dwork.work);
402	struct amd_gpu_scheduler *sched = s_fence->sched;
403	struct amd_sched_fence *entity, *tmp;
404	unsigned long flags;
405
406	DRM_ERROR("[%s] scheduler is timeout!\n", sched->name);
407
408	/* Clean all pending fences */
409	spin_lock_irqsave(&sched->fence_list_lock, flags);
410	list_for_each_entry_safe(entity, tmp, &sched->fence_list, list) {
411		DRM_ERROR("  fence no %d\n", entity->base.seqno);
412		cancel_delayed_work(&entity->dwork);
413		list_del_init(&entity->list);
414		fence_put(&entity->base);
415	}
416	spin_unlock_irqrestore(&sched->fence_list_lock, flags);
417}
418
419static int amd_sched_main(void *param)
420{
421	struct sched_param sparam = {.sched_priority = 1};
422	struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
423	int r, count;
424
425	spin_lock_init(&sched->fence_list_lock);
426	INIT_LIST_HEAD(&sched->fence_list);
427	sched_setscheduler(current, SCHED_FIFO, &sparam);
428
429	while (!kthread_should_stop()) {
430		struct amd_sched_entity *entity;
431		struct amd_sched_fence *s_fence;
432		struct amd_sched_job *sched_job;
433		struct fence *fence;
434		unsigned long flags;
435
436		wait_event_interruptible(sched->wake_up_worker,
437			(entity = amd_sched_select_entity(sched)) ||
438			kthread_should_stop());
439
440		if (!entity)
441			continue;
442
443		sched_job = amd_sched_entity_pop_job(entity);
444		if (!sched_job)
445			continue;
446
447		s_fence = sched_job->s_fence;
448
449		if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
450			INIT_DELAYED_WORK(&s_fence->dwork, amd_sched_fence_work_func);
451			schedule_delayed_work(&s_fence->dwork, sched->timeout);
452			spin_lock_irqsave(&sched->fence_list_lock, flags);
453			list_add_tail(&s_fence->list, &sched->fence_list);
454			spin_unlock_irqrestore(&sched->fence_list_lock, flags);
455		}
456
457		atomic_inc(&sched->hw_rq_count);
458		fence = sched->ops->run_job(sched_job);
459		amd_sched_fence_scheduled(s_fence);
460		if (fence) {
461			r = fence_add_callback(fence, &s_fence->cb,
462					       amd_sched_process_job);
463			if (r == -ENOENT)
464				amd_sched_process_job(fence, &s_fence->cb);
465			else if (r)
466				DRM_ERROR("fence add callback failed (%d)\n", r);
467			fence_put(fence);
468		} else {
469			DRM_ERROR("Failed to run job!\n");
470			amd_sched_process_job(NULL, &s_fence->cb);
471		}
472
473		count = kfifo_out(&entity->job_queue, &sched_job,
474				sizeof(sched_job));
475		WARN_ON(count != sizeof(sched_job));
476		wake_up(&sched->job_scheduled);
477	}
478	return 0;
479}
480
481/**
482 * Init a gpu scheduler instance
483 *
484 * @sched		The pointer to the scheduler
485 * @ops			The backend operations for this scheduler.
486 * @hw_submissions	Number of hw submissions to do.
487 * @name		Name used for debugging
488 *
489 * Return 0 on success, otherwise error code.
490*/
491int amd_sched_init(struct amd_gpu_scheduler *sched,
492		   struct amd_sched_backend_ops *ops,
493		   unsigned hw_submission, long timeout, const char *name)
494{
495	int i;
496	sched->ops = ops;
497	sched->hw_submission_limit = hw_submission;
498	sched->name = name;
499	sched->timeout = timeout;
500	for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++)
501		amd_sched_rq_init(&sched->sched_rq[i]);
502
503	init_waitqueue_head(&sched->wake_up_worker);
504	init_waitqueue_head(&sched->job_scheduled);
505	atomic_set(&sched->hw_rq_count, 0);
506	if (atomic_inc_return(&sched_fence_slab_ref) == 1) {
507		sched_fence_slab = kmem_cache_create(
508			"amd_sched_fence", sizeof(struct amd_sched_fence), 0,
509			SLAB_HWCACHE_ALIGN, NULL);
510		if (!sched_fence_slab)
511			return -ENOMEM;
512	}
513
514	/* Each scheduler will run on a seperate kernel thread */
515	sched->thread = kthread_run(amd_sched_main, sched, sched->name);
516	if (IS_ERR(sched->thread)) {
517		DRM_ERROR("Failed to create scheduler for %s.\n", name);
518		return PTR_ERR(sched->thread);
519	}
520
521	return 0;
522}
523
524/**
525 * Destroy a gpu scheduler
526 *
527 * @sched	The pointer to the scheduler
528 */
529void amd_sched_fini(struct amd_gpu_scheduler *sched)
530{
531	if (sched->thread)
532		kthread_stop(sched->thread);
533	if (atomic_dec_and_test(&sched_fence_slab_ref))
534		kmem_cache_destroy(sched_fence_slab);
535}