Loading...
1/*
2 * Copyright 2016-2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/dma-fence.h>
24#include <linux/spinlock.h>
25#include <linux/atomic.h>
26#include <linux/stacktrace.h>
27#include <linux/sched.h>
28#include <linux/slab.h>
29#include <linux/sched/mm.h>
30#include "amdgpu_amdkfd.h"
31#include "kfd_svm.h"
32
33static const struct dma_fence_ops amdkfd_fence_ops;
34static atomic_t fence_seq = ATOMIC_INIT(0);
35
36/* Eviction Fence
37 * Fence helper functions to deal with KFD memory eviction.
38 * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
39 * evicted unless all the user queues for that process are evicted.
40 *
41 * All the BOs in a process share an eviction fence. When process X wants
42 * to map VRAM memory but TTM can't find enough space, TTM will attempt to
43 * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
44 * by calling ttm_device_funcs->eviction_valuable().
45 *
46 * ttm_device_funcs->eviction_valuable() - will return false if the BO belongs
47 * to process X. Otherwise, it will return true to indicate BO can be
48 * evicted by TTM.
49 *
50 * If ttm_device_funcs->eviction_valuable returns true, then TTM will continue
51 * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
52 * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
53 *
54 * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
55 * nofity when the BO is free to move. fence_add_callback --> enable_signaling
56 * --> amdgpu_amdkfd_fence.enable_signaling
57 *
58 * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
59 * user queues and signal fence. The work item will also start another delayed
60 * work item to restore BOs
61 */
62
63struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
64 struct mm_struct *mm,
65 struct svm_range_bo *svm_bo)
66{
67 struct amdgpu_amdkfd_fence *fence;
68
69 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
70 if (fence == NULL)
71 return NULL;
72
73 /* This reference gets released in amdkfd_fence_release */
74 mmgrab(mm);
75 fence->mm = mm;
76 get_task_comm(fence->timeline_name, current);
77 spin_lock_init(&fence->lock);
78 fence->svm_bo = svm_bo;
79 dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
80 context, atomic_inc_return(&fence_seq));
81
82 return fence;
83}
84
85struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
86{
87 struct amdgpu_amdkfd_fence *fence;
88
89 if (!f)
90 return NULL;
91
92 fence = container_of(f, struct amdgpu_amdkfd_fence, base);
93 if (f->ops == &amdkfd_fence_ops)
94 return fence;
95
96 return NULL;
97}
98
99static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
100{
101 return "amdgpu_amdkfd_fence";
102}
103
104static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
105{
106 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
107
108 return fence->timeline_name;
109}
110
111/**
112 * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
113 * a KFD BO and schedules a job to move the BO.
114 * If fence is already signaled return true.
115 * If fence is not signaled schedule a evict KFD process work item.
116 *
117 * @f: dma_fence
118 */
119static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
120{
121 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
122
123 if (!fence)
124 return false;
125
126 if (dma_fence_is_signaled(f))
127 return true;
128
129 if (!fence->svm_bo) {
130 if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
131 return true;
132 } else {
133 if (!svm_range_schedule_evict_svm_bo(fence))
134 return true;
135 }
136 return false;
137}
138
139/**
140 * amdkfd_fence_release - callback that fence can be freed
141 *
142 * @f: dma_fence
143 *
144 * This function is called when the reference count becomes zero.
145 * Drops the mm_struct reference and RCU schedules freeing up the fence.
146 */
147static void amdkfd_fence_release(struct dma_fence *f)
148{
149 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
150
151 /* Unconditionally signal the fence. The process is getting
152 * terminated.
153 */
154 if (WARN_ON(!fence))
155 return; /* Not an amdgpu_amdkfd_fence */
156
157 mmdrop(fence->mm);
158 kfree_rcu(f, rcu);
159}
160
161/**
162 * amdkfd_fence_check_mm - Check whether to prevent eviction of @f by @mm
163 *
164 * @f: [IN] fence
165 * @mm: [IN] mm that needs to be verified
166 *
167 * Check if @mm is same as that of the fence @f, if same return TRUE else
168 * return FALSE.
169 * For svm bo, which support vram overcommitment, always return FALSE.
170 */
171bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
172{
173 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
174
175 if (!fence)
176 return false;
177 else if (fence->mm == mm && !fence->svm_bo)
178 return true;
179
180 return false;
181}
182
183static const struct dma_fence_ops amdkfd_fence_ops = {
184 .get_driver_name = amdkfd_fence_get_driver_name,
185 .get_timeline_name = amdkfd_fence_get_timeline_name,
186 .enable_signaling = amdkfd_fence_enable_signaling,
187 .release = amdkfd_fence_release,
188};
1/*
2 * Copyright 2016-2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/dma-fence.h>
24#include <linux/spinlock.h>
25#include <linux/atomic.h>
26#include <linux/stacktrace.h>
27#include <linux/sched.h>
28#include <linux/slab.h>
29#include <linux/sched/mm.h>
30#include "amdgpu_amdkfd.h"
31
32static const struct dma_fence_ops amdkfd_fence_ops;
33static atomic_t fence_seq = ATOMIC_INIT(0);
34
35/* Eviction Fence
36 * Fence helper functions to deal with KFD memory eviction.
37 * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
38 * evicted unless all the user queues for that process are evicted.
39 *
40 * All the BOs in a process share an eviction fence. When process X wants
41 * to map VRAM memory but TTM can't find enough space, TTM will attempt to
42 * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
43 * by calling ttm_bo_driver->eviction_valuable().
44 *
45 * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
46 * to process X. Otherwise, it will return true to indicate BO can be
47 * evicted by TTM.
48 *
49 * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
50 * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
51 * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
52 *
53 * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
54 * nofity when the BO is free to move. fence_add_callback --> enable_signaling
55 * --> amdgpu_amdkfd_fence.enable_signaling
56 *
57 * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
58 * user queues and signal fence. The work item will also start another delayed
59 * work item to restore BOs
60 */
61
62struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
63 struct mm_struct *mm)
64{
65 struct amdgpu_amdkfd_fence *fence;
66
67 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
68 if (fence == NULL)
69 return NULL;
70
71 /* This reference gets released in amdkfd_fence_release */
72 mmgrab(mm);
73 fence->mm = mm;
74 get_task_comm(fence->timeline_name, current);
75 spin_lock_init(&fence->lock);
76
77 dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
78 context, atomic_inc_return(&fence_seq));
79
80 return fence;
81}
82
83struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
84{
85 struct amdgpu_amdkfd_fence *fence;
86
87 if (!f)
88 return NULL;
89
90 fence = container_of(f, struct amdgpu_amdkfd_fence, base);
91 if (fence && f->ops == &amdkfd_fence_ops)
92 return fence;
93
94 return NULL;
95}
96
97static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
98{
99 return "amdgpu_amdkfd_fence";
100}
101
102static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
103{
104 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
105
106 return fence->timeline_name;
107}
108
109/**
110 * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
111 * a KFD BO and schedules a job to move the BO.
112 * If fence is already signaled return true.
113 * If fence is not signaled schedule a evict KFD process work item.
114 */
115static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
116{
117 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
118
119 if (!fence)
120 return false;
121
122 if (dma_fence_is_signaled(f))
123 return true;
124
125 if (!kgd2kfd->schedule_evict_and_restore_process(fence->mm, f))
126 return true;
127
128 return false;
129}
130
131/**
132 * amdkfd_fence_release - callback that fence can be freed
133 *
134 * @fence: fence
135 *
136 * This function is called when the reference count becomes zero.
137 * Drops the mm_struct reference and RCU schedules freeing up the fence.
138 */
139static void amdkfd_fence_release(struct dma_fence *f)
140{
141 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
142
143 /* Unconditionally signal the fence. The process is getting
144 * terminated.
145 */
146 if (WARN_ON(!fence))
147 return; /* Not an amdgpu_amdkfd_fence */
148
149 mmdrop(fence->mm);
150 kfree_rcu(f, rcu);
151}
152
153/**
154 * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
155 * if same return TRUE else return FALSE.
156 *
157 * @f: [IN] fence
158 * @mm: [IN] mm that needs to be verified
159 */
160bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
161{
162 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
163
164 if (!fence)
165 return false;
166 else if (fence->mm == mm)
167 return true;
168
169 return false;
170}
171
172static const struct dma_fence_ops amdkfd_fence_ops = {
173 .get_driver_name = amdkfd_fence_get_driver_name,
174 .get_timeline_name = amdkfd_fence_get_timeline_name,
175 .enable_signaling = amdkfd_fence_enable_signaling,
176 .signaled = NULL,
177 .wait = dma_fence_default_wait,
178 .release = amdkfd_fence_release,
179};