1/*
  2 * Copyright 2016-2018 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 */
 22
 23#include <linux/dma-fence.h>
 24#include <linux/spinlock.h>
 25#include <linux/atomic.h>
 26#include <linux/stacktrace.h>
 27#include <linux/sched.h>
 28#include <linux/slab.h>
 29#include <linux/sched/mm.h>
 30#include "amdgpu_amdkfd.h"
 31
 32static const struct dma_fence_ops amdkfd_fence_ops;
 33static atomic_t fence_seq = ATOMIC_INIT(0);
 34
 35/* Eviction Fence
 36 * Fence helper functions to deal with KFD memory eviction.
 37 * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
 38 *  evicted unless all the user queues for that process are evicted.
 39 *
 40 * All the BOs in a process share an eviction fence. When process X wants
 41 * to map VRAM memory but TTM can't find enough space, TTM will attempt to
 42 * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
 43 * by calling ttm_bo_driver->eviction_valuable().
 44 *
 45 * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
 46 *  to process X. Otherwise, it will return true to indicate BO can be
 47 *  evicted by TTM.
 48 *
 49 * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
 50 * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
 51 * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
 52 *
 53 * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
 54 *  nofity when the BO is free to move. fence_add_callback --> enable_signaling
 55 *  --> amdgpu_amdkfd_fence.enable_signaling
 56 *
 57 * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
 58 * user queues and signal fence. The work item will also start another delayed
 59 * work item to restore BOs
 60 */
 61
 62struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 63						       struct mm_struct *mm)
 64{
 65	struct amdgpu_amdkfd_fence *fence;
 66
 67	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 68	if (fence == NULL)
 69		return NULL;
 70
 71	/* This reference gets released in amdkfd_fence_release */
 72	mmgrab(mm);
 73	fence->mm = mm;
 74	get_task_comm(fence->timeline_name, current);
 75	spin_lock_init(&fence->lock);
 76
 77	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
 78		   context, atomic_inc_return(&fence_seq));
 79
 80	return fence;
 81}
 82
 83struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
 84{
 85	struct amdgpu_amdkfd_fence *fence;
 86
 87	if (!f)
 88		return NULL;
 89
 90	fence = container_of(f, struct amdgpu_amdkfd_fence, base);
 91	if (fence && f->ops == &amdkfd_fence_ops)
 92		return fence;
 93
 94	return NULL;
 95}
 96
 97static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
 98{
 99	return "amdgpu_amdkfd_fence";
100}
101
102static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
103{
104	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
105
106	return fence->timeline_name;
107}
108
109/**
110 * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
111 *  a KFD BO and schedules a job to move the BO.
112 *  If fence is already signaled return true.
113 *  If fence is not signaled schedule a evict KFD process work item.
114 */
115static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
116{
117	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
118
119	if (!fence)
120		return false;
121
122	if (dma_fence_is_signaled(f))
123		return true;
124
125	if (!kgd2kfd->schedule_evict_and_restore_process(fence->mm, f))
126		return true;
127
128	return false;
129}
130
131/**
132 * amdkfd_fence_release - callback that fence can be freed
133 *
134 * @fence: fence
135 *
136 * This function is called when the reference count becomes zero.
137 * Drops the mm_struct reference and RCU schedules freeing up the fence.
138 */
139static void amdkfd_fence_release(struct dma_fence *f)
140{
141	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
142
143	/* Unconditionally signal the fence. The process is getting
144	 * terminated.
145	 */
146	if (WARN_ON(!fence))
147		return; /* Not an amdgpu_amdkfd_fence */
148
149	mmdrop(fence->mm);
150	kfree_rcu(f, rcu);
151}
152
153/**
154 * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
155 *  if same return TRUE else return FALSE.
156 *
157 * @f: [IN] fence
158 * @mm: [IN] mm that needs to be verified
159 */
160bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
161{
162	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
163
164	if (!fence)
165		return false;
166	else if (fence->mm == mm)
167		return true;
168
169	return false;
170}
171
172static const struct dma_fence_ops amdkfd_fence_ops = {
173	.get_driver_name = amdkfd_fence_get_driver_name,
174	.get_timeline_name = amdkfd_fence_get_timeline_name,
175	.enable_signaling = amdkfd_fence_enable_signaling,
176	.signaled = NULL,
177	.wait = dma_fence_default_wait,
178	.release = amdkfd_fence_release,
179};