1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/slab.h>
28 #include <linux/amd-iommu.h>
29 #include <linux/notifier.h>
30 #include <linux/compat.h>
31 
32 struct mm_struct;
33 
34 #include "kfd_priv.h"
35 #include "kfd_dbgmgr.h"
36 
37 /*
38  * Initial size for the array of queues.
39  * The allocated size is doubled each time
40  * it is exceeded up to MAX_PROCESS_QUEUES.
41  */
42 #define INITIAL_QUEUE_ARRAY_SIZE 16
43 
44 /*
45  * List of struct kfd_process (field kfd_process).
46  * Unique/indexed by mm_struct*
47  */
48 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
49 static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
50 static DEFINE_MUTEX(kfd_processes_mutex);
51 
52 DEFINE_STATIC_SRCU(kfd_processes_srcu);
53 
54 static struct workqueue_struct *kfd_process_wq;
55 
56 struct kfd_process_release_work {
57 	struct work_struct kfd_work;
58 	struct kfd_process *p;
59 };
60 
61 static struct kfd_process *find_process(const struct task_struct *thread);
62 static struct kfd_process *create_process(const struct task_struct *thread);
63 
64 void kfd_process_create_wq(void)
65 {
66 	if (!kfd_process_wq)
67 		kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
68 }
69 
70 void kfd_process_destroy_wq(void)
71 {
72 	if (kfd_process_wq) {
73 		destroy_workqueue(kfd_process_wq);
74 		kfd_process_wq = NULL;
75 	}
76 }
77 
78 struct kfd_process *kfd_create_process(const struct task_struct *thread)
79 {
80 	struct kfd_process *process;
81 
82 	BUG_ON(!kfd_process_wq);
83 
84 	if (thread->mm == NULL)
85 		return ERR_PTR(-EINVAL);
86 
87 	/* Only the pthreads threading model is supported. */
88 	if (thread->group_leader->mm != thread->mm)
89 		return ERR_PTR(-EINVAL);
90 
91 	/* Take mmap_sem because we call __mmu_notifier_register inside */
92 	down_write(&thread->mm->mmap_sem);
93 
94 	/*
95 	 * take kfd processes mutex before starting of process creation
96 	 * so there won't be a case where two threads of the same process
97 	 * create two kfd_process structures
98 	 */
99 	mutex_lock(&kfd_processes_mutex);
100 
101 	/* A prior open of /dev/kfd could have already created the process. */
102 	process = find_process(thread);
103 	if (process)
104 		pr_debug("kfd: process already found\n");
105 
106 	if (!process)
107 		process = create_process(thread);
108 
109 	mutex_unlock(&kfd_processes_mutex);
110 
111 	up_write(&thread->mm->mmap_sem);
112 
113 	return process;
114 }
115 
116 struct kfd_process *kfd_get_process(const struct task_struct *thread)
117 {
118 	struct kfd_process *process;
119 
120 	if (thread->mm == NULL)
121 		return ERR_PTR(-EINVAL);
122 
123 	/* Only the pthreads threading model is supported. */
124 	if (thread->group_leader->mm != thread->mm)
125 		return ERR_PTR(-EINVAL);
126 
127 	process = find_process(thread);
128 
129 	return process;
130 }
131 
132 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
133 {
134 	struct kfd_process *process;
135 
136 	hash_for_each_possible_rcu(kfd_processes_table, process,
137 					kfd_processes, (uintptr_t)mm)
138 		if (process->mm == mm)
139 			return process;
140 
141 	return NULL;
142 }
143 
144 static struct kfd_process *find_process(const struct task_struct *thread)
145 {
146 	struct kfd_process *p;
147 	int idx;
148 
149 	idx = srcu_read_lock(&kfd_processes_srcu);
150 	p = find_process_by_mm(thread->mm);
151 	srcu_read_unlock(&kfd_processes_srcu, idx);
152 
153 	return p;
154 }
155 
156 static void kfd_process_wq_release(struct work_struct *work)
157 {
158 	struct kfd_process_release_work *my_work;
159 	struct kfd_process_device *pdd, *temp;
160 	struct kfd_process *p;
161 
162 	my_work = (struct kfd_process_release_work *) work;
163 
164 	p = my_work->p;
165 
166 	pr_debug("Releasing process (pasid %d) in workqueue\n",
167 			p->pasid);
168 
169 	mutex_lock(&p->mutex);
170 
171 	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
172 							per_device_list) {
173 		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
174 				pdd->dev->id, p->pasid);
175 
176 		if (pdd->reset_wavefronts)
177 			dbgdev_wave_reset_wavefronts(pdd->dev, p);
178 
179 		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
180 		list_del(&pdd->per_device_list);
181 
182 		kfree(pdd);
183 	}
184 
185 	kfd_event_free_process(p);
186 
187 	kfd_pasid_free(p->pasid);
188 
189 	mutex_unlock(&p->mutex);
190 
191 	mutex_destroy(&p->mutex);
192 
193 	kfree(p->queues);
194 
195 	kfree(p);
196 
197 	kfree(work);
198 }
199 
200 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
201 {
202 	struct kfd_process_release_work *work;
203 	struct kfd_process *p;
204 
205 	BUG_ON(!kfd_process_wq);
206 
207 	p = container_of(rcu, struct kfd_process, rcu);
208 	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
209 
210 	mmdrop(p->mm);
211 
212 	work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
213 
214 	if (work) {
215 		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
216 		work->p = p;
217 		queue_work(kfd_process_wq, (struct work_struct *) work);
218 	}
219 }
220 
221 static void kfd_process_notifier_release(struct mmu_notifier *mn,
222 					struct mm_struct *mm)
223 {
224 	struct kfd_process *p;
225 	struct kfd_process_device *pdd = NULL;
226 
227 	/*
228 	 * The kfd_process structure can not be free because the
229 	 * mmu_notifier srcu is read locked
230 	 */
231 	p = container_of(mn, struct kfd_process, mmu_notifier);
232 	BUG_ON(p->mm != mm);
233 
234 	mutex_lock(&kfd_processes_mutex);
235 	hash_del_rcu(&p->kfd_processes);
236 	mutex_unlock(&kfd_processes_mutex);
237 	synchronize_srcu(&kfd_processes_srcu);
238 
239 	mutex_lock(&p->mutex);
240 
241 	/* In case our notifier is called before IOMMU notifier */
242 	pqm_uninit(&p->pqm);
243 
244 	/* Iterate over all process device data structure and check
245 	 * if we should delete debug managers and reset all wavefronts
246 	 */
247 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
248 		if ((pdd->dev->dbgmgr) &&
249 				(pdd->dev->dbgmgr->pasid == p->pasid))
250 			kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
251 
252 		if (pdd->reset_wavefronts) {
253 			pr_warn("amdkfd: Resetting all wave fronts\n");
254 			dbgdev_wave_reset_wavefronts(pdd->dev, p);
255 			pdd->reset_wavefronts = false;
256 		}
257 	}
258 
259 	mutex_unlock(&p->mutex);
260 
261 	/*
262 	 * Because we drop mm_count inside kfd_process_destroy_delayed
263 	 * and because the mmu_notifier_unregister function also drop
264 	 * mm_count we need to take an extra count here.
265 	 */
266 	mmgrab(p->mm);
267 	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
268 	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
269 }
270 
271 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
272 	.release = kfd_process_notifier_release,
273 };
274 
275 static struct kfd_process *create_process(const struct task_struct *thread)
276 {
277 	struct kfd_process *process;
278 	int err = -ENOMEM;
279 
280 	process = kzalloc(sizeof(*process), GFP_KERNEL);
281 
282 	if (!process)
283 		goto err_alloc_process;
284 
285 	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
286 					sizeof(process->queues[0]), GFP_KERNEL);
287 	if (!process->queues)
288 		goto err_alloc_queues;
289 
290 	process->pasid = kfd_pasid_alloc();
291 	if (process->pasid == 0)
292 		goto err_alloc_pasid;
293 
294 	mutex_init(&process->mutex);
295 
296 	process->mm = thread->mm;
297 
298 	/* register notifier */
299 	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
300 	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
301 	if (err)
302 		goto err_mmu_notifier;
303 
304 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
305 			(uintptr_t)process->mm);
306 
307 	process->lead_thread = thread->group_leader;
308 
309 	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
310 
311 	INIT_LIST_HEAD(&process->per_device_data);
312 
313 	kfd_event_init_process(process);
314 
315 	err = pqm_init(&process->pqm, process);
316 	if (err != 0)
317 		goto err_process_pqm_init;
318 
319 	/* init process apertures*/
320 	process->is_32bit_user_mode = in_compat_syscall();
321 	err = kfd_init_apertures(process);
322 	if (err != 0)
323 		goto err_init_apertures;
324 
325 	return process;
326 
327 err_init_apertures:
328 	pqm_uninit(&process->pqm);
329 err_process_pqm_init:
330 	hash_del_rcu(&process->kfd_processes);
331 	synchronize_rcu();
332 	mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
333 err_mmu_notifier:
334 	mutex_destroy(&process->mutex);
335 	kfd_pasid_free(process->pasid);
336 err_alloc_pasid:
337 	kfree(process->queues);
338 err_alloc_queues:
339 	kfree(process);
340 err_alloc_process:
341 	return ERR_PTR(err);
342 }
343 
344 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
345 							struct kfd_process *p)
346 {
347 	struct kfd_process_device *pdd = NULL;
348 
349 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
350 		if (pdd->dev == dev)
351 			break;
352 
353 	return pdd;
354 }
355 
356 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
357 							struct kfd_process *p)
358 {
359 	struct kfd_process_device *pdd = NULL;
360 
361 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
362 	if (pdd != NULL) {
363 		pdd->dev = dev;
364 		INIT_LIST_HEAD(&pdd->qpd.queues_list);
365 		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
366 		pdd->qpd.dqm = dev->dqm;
367 		pdd->reset_wavefronts = false;
368 		list_add(&pdd->per_device_list, &p->per_device_data);
369 	}
370 
371 	return pdd;
372 }
373 
374 /*
375  * Direct the IOMMU to bind the process (specifically the pasid->mm)
376  * to the device.
377  * Unbinding occurs when the process dies or the device is removed.
378  *
379  * Assumes that the process lock is held.
380  */
381 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
382 							struct kfd_process *p)
383 {
384 	struct kfd_process_device *pdd;
385 	int err;
386 
387 	pdd = kfd_get_process_device_data(dev, p);
388 	if (!pdd) {
389 		pr_err("Process device data doesn't exist\n");
390 		return ERR_PTR(-ENOMEM);
391 	}
392 
393 	if (pdd->bound)
394 		return pdd;
395 
396 	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
397 	if (err < 0)
398 		return ERR_PTR(err);
399 
400 	pdd->bound = true;
401 
402 	return pdd;
403 }
404 
405 void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
406 {
407 	struct kfd_process *p;
408 	struct kfd_process_device *pdd;
409 
410 	BUG_ON(dev == NULL);
411 
412 	/*
413 	 * Look for the process that matches the pasid. If there is no such
414 	 * process, we either released it in amdkfd's own notifier, or there
415 	 * is a bug. Unfortunately, there is no way to tell...
416 	 */
417 	p = kfd_lookup_process_by_pasid(pasid);
418 	if (!p)
419 		return;
420 
421 	pr_debug("Unbinding process %d from IOMMU\n", pasid);
422 
423 	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
424 		kfd_dbgmgr_destroy(dev->dbgmgr);
425 
426 	pqm_uninit(&p->pqm);
427 
428 	pdd = kfd_get_process_device_data(dev, p);
429 
430 	if (!pdd) {
431 		mutex_unlock(&p->mutex);
432 		return;
433 	}
434 
435 	if (pdd->reset_wavefronts) {
436 		dbgdev_wave_reset_wavefronts(pdd->dev, p);
437 		pdd->reset_wavefronts = false;
438 	}
439 
440 	/*
441 	 * Just mark pdd as unbound, because we still need it
442 	 * to call amd_iommu_unbind_pasid() in when the
443 	 * process exits.
444 	 * We don't call amd_iommu_unbind_pasid() here
445 	 * because the IOMMU called us.
446 	 */
447 	pdd->bound = false;
448 
449 	mutex_unlock(&p->mutex);
450 }
451 
452 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
453 {
454 	return list_first_entry(&p->per_device_data,
455 				struct kfd_process_device,
456 				per_device_list);
457 }
458 
459 struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
460 						struct kfd_process_device *pdd)
461 {
462 	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
463 		return NULL;
464 	return list_next_entry(pdd, per_device_list);
465 }
466 
467 bool kfd_has_process_device_data(struct kfd_process *p)
468 {
469 	return !(list_empty(&p->per_device_data));
470 }
471 
472 /* This returns with process->mutex locked. */
473 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
474 {
475 	struct kfd_process *p;
476 	unsigned int temp;
477 
478 	int idx = srcu_read_lock(&kfd_processes_srcu);
479 
480 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
481 		if (p->pasid == pasid) {
482 			mutex_lock(&p->mutex);
483 			break;
484 		}
485 	}
486 
487 	srcu_read_unlock(&kfd_processes_srcu, idx);
488 
489 	return p;
490 }
491