1/*
  2 * Copyright 2014 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 */
 22
 23#include <linux/mutex.h>
 24#include <linux/log2.h>
 25#include <linux/sched.h>
 26#include <linux/slab.h>
 27#include <linux/amd-iommu.h>
 28#include <linux/notifier.h>
 29#include <linux/compat.h>
 30
 31struct mm_struct;
 32
 33#include "kfd_priv.h"
 34#include "kfd_dbgmgr.h"
 35
 36/*
 37 * Initial size for the array of queues.
 38 * The allocated size is doubled each time
 39 * it is exceeded up to MAX_PROCESS_QUEUES.
 40 */
 41#define INITIAL_QUEUE_ARRAY_SIZE 16
 42
 43/*
 44 * List of struct kfd_process (field kfd_process).
 45 * Unique/indexed by mm_struct*
 46 */
 47#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
 48static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
 49static DEFINE_MUTEX(kfd_processes_mutex);
 50
 51DEFINE_STATIC_SRCU(kfd_processes_srcu);
 52
 53static struct workqueue_struct *kfd_process_wq;
 54
 55struct kfd_process_release_work {
 56	struct work_struct kfd_work;
 57	struct kfd_process *p;
 58};
 59
 60static struct kfd_process *find_process(const struct task_struct *thread);
 61static struct kfd_process *create_process(const struct task_struct *thread);
 62
 63void kfd_process_create_wq(void)
 64{
 65	if (!kfd_process_wq)
 66		kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
 67}
 68
 69void kfd_process_destroy_wq(void)
 70{
 71	if (kfd_process_wq) {
 72		destroy_workqueue(kfd_process_wq);
 73		kfd_process_wq = NULL;
 74	}
 75}
 76
 77struct kfd_process *kfd_create_process(const struct task_struct *thread)
 78{
 79	struct kfd_process *process;
 80
 81	BUG_ON(!kfd_process_wq);
 82
 83	if (thread->mm == NULL)
 84		return ERR_PTR(-EINVAL);
 85
 86	/* Only the pthreads threading model is supported. */
 87	if (thread->group_leader->mm != thread->mm)
 88		return ERR_PTR(-EINVAL);
 89
 90	/* Take mmap_sem because we call __mmu_notifier_register inside */
 91	down_write(&thread->mm->mmap_sem);
 92
 93	/*
 94	 * take kfd processes mutex before starting of process creation
 95	 * so there won't be a case where two threads of the same process
 96	 * create two kfd_process structures
 97	 */
 98	mutex_lock(&kfd_processes_mutex);
 99
100	/* A prior open of /dev/kfd could have already created the process. */
101	process = find_process(thread);
102	if (process)
103		pr_debug("kfd: process already found\n");
104
105	if (!process)
106		process = create_process(thread);
107
108	mutex_unlock(&kfd_processes_mutex);
109
110	up_write(&thread->mm->mmap_sem);
111
112	return process;
113}
114
115struct kfd_process *kfd_get_process(const struct task_struct *thread)
116{
117	struct kfd_process *process;
118
119	if (thread->mm == NULL)
120		return ERR_PTR(-EINVAL);
121
122	/* Only the pthreads threading model is supported. */
123	if (thread->group_leader->mm != thread->mm)
124		return ERR_PTR(-EINVAL);
125
126	process = find_process(thread);
127
128	return process;
129}
130
131static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
132{
133	struct kfd_process *process;
134
135	hash_for_each_possible_rcu(kfd_processes_table, process,
136					kfd_processes, (uintptr_t)mm)
137		if (process->mm == mm)
138			return process;
139
140	return NULL;
141}
142
143static struct kfd_process *find_process(const struct task_struct *thread)
144{
145	struct kfd_process *p;
146	int idx;
147
148	idx = srcu_read_lock(&kfd_processes_srcu);
149	p = find_process_by_mm(thread->mm);
150	srcu_read_unlock(&kfd_processes_srcu, idx);
151
152	return p;
153}
154
155static void kfd_process_wq_release(struct work_struct *work)
156{
157	struct kfd_process_release_work *my_work;
158	struct kfd_process_device *pdd, *temp;
159	struct kfd_process *p;
160
161	my_work = (struct kfd_process_release_work *) work;
162
163	p = my_work->p;
164
165	pr_debug("Releasing process (pasid %d) in workqueue\n",
166			p->pasid);
167
168	mutex_lock(&p->mutex);
169
170	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
171							per_device_list) {
172		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
173				pdd->dev->id, p->pasid);
174
175		if (pdd->reset_wavefronts)
176			dbgdev_wave_reset_wavefronts(pdd->dev, p);
177
178		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
179		list_del(&pdd->per_device_list);
180
181		kfree(pdd);
182	}
183
184	kfd_event_free_process(p);
185
186	kfd_pasid_free(p->pasid);
187
188	mutex_unlock(&p->mutex);
189
190	mutex_destroy(&p->mutex);
191
192	kfree(p->queues);
193
194	kfree(p);
195
196	kfree(work);
197}
198
199static void kfd_process_destroy_delayed(struct rcu_head *rcu)
200{
201	struct kfd_process_release_work *work;
202	struct kfd_process *p;
203
204	BUG_ON(!kfd_process_wq);
205
206	p = container_of(rcu, struct kfd_process, rcu);
207	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
208
209	mmdrop(p->mm);
210
211	work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
212
213	if (work) {
214		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
215		work->p = p;
216		queue_work(kfd_process_wq, (struct work_struct *) work);
217	}
218}
219
220static void kfd_process_notifier_release(struct mmu_notifier *mn,
221					struct mm_struct *mm)
222{
223	struct kfd_process *p;
224	struct kfd_process_device *pdd = NULL;
225
226	/*
227	 * The kfd_process structure can not be free because the
228	 * mmu_notifier srcu is read locked
229	 */
230	p = container_of(mn, struct kfd_process, mmu_notifier);
231	BUG_ON(p->mm != mm);
232
233	mutex_lock(&kfd_processes_mutex);
234	hash_del_rcu(&p->kfd_processes);
235	mutex_unlock(&kfd_processes_mutex);
236	synchronize_srcu(&kfd_processes_srcu);
237
238	mutex_lock(&p->mutex);
239
240	/* In case our notifier is called before IOMMU notifier */
241	pqm_uninit(&p->pqm);
242
243	/* Iterate over all process device data structure and check
244	 * if we should delete debug managers and reset all wavefronts
245	 */
246	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
247		if ((pdd->dev->dbgmgr) &&
248				(pdd->dev->dbgmgr->pasid == p->pasid))
249			kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
250
251		if (pdd->reset_wavefronts) {
252			pr_warn("amdkfd: Resetting all wave fronts\n");
253			dbgdev_wave_reset_wavefronts(pdd->dev, p);
254			pdd->reset_wavefronts = false;
255		}
256	}
257
258	mutex_unlock(&p->mutex);
259
260	/*
261	 * Because we drop mm_count inside kfd_process_destroy_delayed
262	 * and because the mmu_notifier_unregister function also drop
263	 * mm_count we need to take an extra count here.
264	 */
265	atomic_inc(&p->mm->mm_count);
266	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
267	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
268}
269
270static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
271	.release = kfd_process_notifier_release,
272};
273
274static struct kfd_process *create_process(const struct task_struct *thread)
275{
276	struct kfd_process *process;
277	int err = -ENOMEM;
278
279	process = kzalloc(sizeof(*process), GFP_KERNEL);
280
281	if (!process)
282		goto err_alloc_process;
283
284	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
285					sizeof(process->queues[0]), GFP_KERNEL);
286	if (!process->queues)
287		goto err_alloc_queues;
288
289	process->pasid = kfd_pasid_alloc();
290	if (process->pasid == 0)
291		goto err_alloc_pasid;
292
293	mutex_init(&process->mutex);
294
295	process->mm = thread->mm;
296
297	/* register notifier */
298	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
299	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
300	if (err)
301		goto err_mmu_notifier;
302
303	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
304			(uintptr_t)process->mm);
305
306	process->lead_thread = thread->group_leader;
307
308	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
309
310	INIT_LIST_HEAD(&process->per_device_data);
311
312	kfd_event_init_process(process);
313
314	err = pqm_init(&process->pqm, process);
315	if (err != 0)
316		goto err_process_pqm_init;
317
318	/* init process apertures*/
319	process->is_32bit_user_mode = in_compat_syscall();
320	if (kfd_init_apertures(process) != 0)
321		goto err_init_apretures;
322
323	return process;
324
325err_init_apretures:
326	pqm_uninit(&process->pqm);
327err_process_pqm_init:
328	hash_del_rcu(&process->kfd_processes);
329	synchronize_rcu();
330	mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
331err_mmu_notifier:
332	mutex_destroy(&process->mutex);
333	kfd_pasid_free(process->pasid);
334err_alloc_pasid:
335	kfree(process->queues);
336err_alloc_queues:
337	kfree(process);
338err_alloc_process:
339	return ERR_PTR(err);
340}
341
342struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
343							struct kfd_process *p)
344{
345	struct kfd_process_device *pdd = NULL;
346
347	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
348		if (pdd->dev == dev)
349			break;
350
351	return pdd;
352}
353
354struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
355							struct kfd_process *p)
356{
357	struct kfd_process_device *pdd = NULL;
358
359	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
360	if (pdd != NULL) {
361		pdd->dev = dev;
362		INIT_LIST_HEAD(&pdd->qpd.queues_list);
363		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
364		pdd->qpd.dqm = dev->dqm;
365		pdd->reset_wavefronts = false;
366		list_add(&pdd->per_device_list, &p->per_device_data);
367	}
368
369	return pdd;
370}
371
372/*
373 * Direct the IOMMU to bind the process (specifically the pasid->mm)
374 * to the device.
375 * Unbinding occurs when the process dies or the device is removed.
376 *
377 * Assumes that the process lock is held.
378 */
379struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
380							struct kfd_process *p)
381{
382	struct kfd_process_device *pdd;
383	int err;
384
385	pdd = kfd_get_process_device_data(dev, p);
386	if (!pdd) {
387		pr_err("Process device data doesn't exist\n");
388		return ERR_PTR(-ENOMEM);
389	}
390
391	if (pdd->bound)
392		return pdd;
393
394	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
395	if (err < 0)
396		return ERR_PTR(err);
397
398	pdd->bound = true;
399
400	return pdd;
401}
402
403void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
404{
405	struct kfd_process *p;
406	struct kfd_process_device *pdd;
407
408	BUG_ON(dev == NULL);
409
410	/*
411	 * Look for the process that matches the pasid. If there is no such
412	 * process, we either released it in amdkfd's own notifier, or there
413	 * is a bug. Unfortunately, there is no way to tell...
414	 */
415	p = kfd_lookup_process_by_pasid(pasid);
416	if (!p)
417		return;
418
419	pr_debug("Unbinding process %d from IOMMU\n", pasid);
420
421	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
422		kfd_dbgmgr_destroy(dev->dbgmgr);
423
424	pqm_uninit(&p->pqm);
425
426	pdd = kfd_get_process_device_data(dev, p);
427
428	if (!pdd) {
429		mutex_unlock(&p->mutex);
430		return;
431	}
432
433	if (pdd->reset_wavefronts) {
434		dbgdev_wave_reset_wavefronts(pdd->dev, p);
435		pdd->reset_wavefronts = false;
436	}
437
438	/*
439	 * Just mark pdd as unbound, because we still need it
440	 * to call amd_iommu_unbind_pasid() in when the
441	 * process exits.
442	 * We don't call amd_iommu_unbind_pasid() here
443	 * because the IOMMU called us.
444	 */
445	pdd->bound = false;
446
447	mutex_unlock(&p->mutex);
448}
449
450struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
451{
452	return list_first_entry(&p->per_device_data,
453				struct kfd_process_device,
454				per_device_list);
455}
456
457struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
458						struct kfd_process_device *pdd)
459{
460	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
461		return NULL;
462	return list_next_entry(pdd, per_device_list);
463}
464
465bool kfd_has_process_device_data(struct kfd_process *p)
466{
467	return !(list_empty(&p->per_device_data));
468}
469
470/* This returns with process->mutex locked. */
471struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
472{
473	struct kfd_process *p;
474	unsigned int temp;
475
476	int idx = srcu_read_lock(&kfd_processes_srcu);
477
478	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
479		if (p->pasid == pasid) {
480			mutex_lock(&p->mutex);
481			break;
482		}
483	}
484
485	srcu_read_unlock(&kfd_processes_srcu, idx);
486
487	return p;
488}