1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/slab.h>
28 #include <linux/amd-iommu.h>
29 #include <linux/notifier.h>
30 #include <linux/compat.h>
31 
32 struct mm_struct;
33 
34 #include "kfd_priv.h"
35 #include "kfd_dbgmgr.h"
36 
37 /*
38  * Initial size for the array of queues.
39  * The allocated size is doubled each time
40  * it is exceeded up to MAX_PROCESS_QUEUES.
41  */
42 #define INITIAL_QUEUE_ARRAY_SIZE 16
43 
44 /*
45  * List of struct kfd_process (field kfd_process).
46  * Unique/indexed by mm_struct*
47  */
48 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
49 static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
50 static DEFINE_MUTEX(kfd_processes_mutex);
51 
52 DEFINE_STATIC_SRCU(kfd_processes_srcu);
53 
54 static struct workqueue_struct *kfd_process_wq;
55 
56 struct kfd_process_release_work {
57 	struct work_struct kfd_work;
58 	struct kfd_process *p;
59 };
60 
61 static struct kfd_process *find_process(const struct task_struct *thread);
62 static struct kfd_process *create_process(const struct task_struct *thread);
63 
64 void kfd_process_create_wq(void)
65 {
66 	if (!kfd_process_wq)
67 		kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
68 }
69 
70 void kfd_process_destroy_wq(void)
71 {
72 	if (kfd_process_wq) {
73 		destroy_workqueue(kfd_process_wq);
74 		kfd_process_wq = NULL;
75 	}
76 }
77 
78 struct kfd_process *kfd_create_process(const struct task_struct *thread)
79 {
80 	struct kfd_process *process;
81 
82 	BUG_ON(!kfd_process_wq);
83 
84 	if (thread->mm == NULL)
85 		return ERR_PTR(-EINVAL);
86 
87 	/* Only the pthreads threading model is supported. */
88 	if (thread->group_leader->mm != thread->mm)
89 		return ERR_PTR(-EINVAL);
90 
91 	/* Take mmap_sem because we call __mmu_notifier_register inside */
92 	down_write(&thread->mm->mmap_sem);
93 
94 	/*
95 	 * take kfd processes mutex before starting of process creation
96 	 * so there won't be a case where two threads of the same process
97 	 * create two kfd_process structures
98 	 */
99 	mutex_lock(&kfd_processes_mutex);
100 
101 	/* A prior open of /dev/kfd could have already created the process. */
102 	process = find_process(thread);
103 	if (process)
104 		pr_debug("kfd: process already found\n");
105 
106 	if (!process)
107 		process = create_process(thread);
108 
109 	mutex_unlock(&kfd_processes_mutex);
110 
111 	up_write(&thread->mm->mmap_sem);
112 
113 	return process;
114 }
115 
116 struct kfd_process *kfd_get_process(const struct task_struct *thread)
117 {
118 	struct kfd_process *process;
119 
120 	if (thread->mm == NULL)
121 		return ERR_PTR(-EINVAL);
122 
123 	/* Only the pthreads threading model is supported. */
124 	if (thread->group_leader->mm != thread->mm)
125 		return ERR_PTR(-EINVAL);
126 
127 	process = find_process(thread);
128 
129 	return process;
130 }
131 
132 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
133 {
134 	struct kfd_process *process;
135 
136 	hash_for_each_possible_rcu(kfd_processes_table, process,
137 					kfd_processes, (uintptr_t)mm)
138 		if (process->mm == mm)
139 			return process;
140 
141 	return NULL;
142 }
143 
144 static struct kfd_process *find_process(const struct task_struct *thread)
145 {
146 	struct kfd_process *p;
147 	int idx;
148 
149 	idx = srcu_read_lock(&kfd_processes_srcu);
150 	p = find_process_by_mm(thread->mm);
151 	srcu_read_unlock(&kfd_processes_srcu, idx);
152 
153 	return p;
154 }
155 
156 static void kfd_process_wq_release(struct work_struct *work)
157 {
158 	struct kfd_process_release_work *my_work;
159 	struct kfd_process_device *pdd, *temp;
160 	struct kfd_process *p;
161 
162 	my_work = (struct kfd_process_release_work *) work;
163 
164 	p = my_work->p;
165 
166 	pr_debug("Releasing process (pasid %d) in workqueue\n",
167 			p->pasid);
168 
169 	mutex_lock(&p->mutex);
170 
171 	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
172 							per_device_list) {
173 		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
174 				pdd->dev->id, p->pasid);
175 
176 		if (pdd->reset_wavefronts)
177 			dbgdev_wave_reset_wavefronts(pdd->dev, p);
178 
179 		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
180 		list_del(&pdd->per_device_list);
181 
182 		kfree(pdd);
183 	}
184 
185 	kfd_event_free_process(p);
186 
187 	kfd_pasid_free(p->pasid);
188 
189 	mutex_unlock(&p->mutex);
190 
191 	mutex_destroy(&p->mutex);
192 
193 	kfree(p->queues);
194 
195 	kfree(p);
196 
197 	kfree(work);
198 }
199 
200 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
201 {
202 	struct kfd_process_release_work *work;
203 	struct kfd_process *p;
204 
205 	BUG_ON(!kfd_process_wq);
206 
207 	p = container_of(rcu, struct kfd_process, rcu);
208 	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
209 
210 	mmdrop(p->mm);
211 
212 	work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
213 
214 	if (work) {
215 		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
216 		work->p = p;
217 		queue_work(kfd_process_wq, (struct work_struct *) work);
218 	}
219 }
220 
221 static void kfd_process_notifier_release(struct mmu_notifier *mn,
222 					struct mm_struct *mm)
223 {
224 	struct kfd_process *p;
225 	struct kfd_process_device *pdd = NULL;
226 
227 	/*
228 	 * The kfd_process structure can not be free because the
229 	 * mmu_notifier srcu is read locked
230 	 */
231 	p = container_of(mn, struct kfd_process, mmu_notifier);
232 	BUG_ON(p->mm != mm);
233 
234 	mutex_lock(&kfd_processes_mutex);
235 	hash_del_rcu(&p->kfd_processes);
236 	mutex_unlock(&kfd_processes_mutex);
237 	synchronize_srcu(&kfd_processes_srcu);
238 
239 	mutex_lock(&p->mutex);
240 
241 	/* In case our notifier is called before IOMMU notifier */
242 	pqm_uninit(&p->pqm);
243 
244 	/* Iterate over all process device data structure and check
245 	 * if we should delete debug managers and reset all wavefronts
246 	 */
247 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
248 		if ((pdd->dev->dbgmgr) &&
249 				(pdd->dev->dbgmgr->pasid == p->pasid))
250 			kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
251 
252 		if (pdd->reset_wavefronts) {
253 			pr_warn("amdkfd: Resetting all wave fronts\n");
254 			dbgdev_wave_reset_wavefronts(pdd->dev, p);
255 			pdd->reset_wavefronts = false;
256 		}
257 	}
258 
259 	mutex_unlock(&p->mutex);
260 
261 	/*
262 	 * Because we drop mm_count inside kfd_process_destroy_delayed
263 	 * and because the mmu_notifier_unregister function also drop
264 	 * mm_count we need to take an extra count here.
265 	 */
266 	mmgrab(p->mm);
267 	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
268 	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
269 }
270 
271 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
272 	.release = kfd_process_notifier_release,
273 };
274 
275 static struct kfd_process *create_process(const struct task_struct *thread)
276 {
277 	struct kfd_process *process;
278 	int err = -ENOMEM;
279 
280 	process = kzalloc(sizeof(*process), GFP_KERNEL);
281 
282 	if (!process)
283 		goto err_alloc_process;
284 
285 	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
286 					sizeof(process->queues[0]), GFP_KERNEL);
287 	if (!process->queues)
288 		goto err_alloc_queues;
289 
290 	process->pasid = kfd_pasid_alloc();
291 	if (process->pasid == 0)
292 		goto err_alloc_pasid;
293 
294 	mutex_init(&process->mutex);
295 
296 	process->mm = thread->mm;
297 
298 	/* register notifier */
299 	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
300 	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
301 	if (err)
302 		goto err_mmu_notifier;
303 
304 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
305 			(uintptr_t)process->mm);
306 
307 	process->lead_thread = thread->group_leader;
308 
309 	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
310 
311 	INIT_LIST_HEAD(&process->per_device_data);
312 
313 	kfd_event_init_process(process);
314 
315 	err = pqm_init(&process->pqm, process);
316 	if (err != 0)
317 		goto err_process_pqm_init;
318 
319 	/* init process apertures*/
320 	process->is_32bit_user_mode = in_compat_syscall();
321 	if (kfd_init_apertures(process) != 0)
322 		goto err_init_apretures;
323 
324 	return process;
325 
326 err_init_apretures:
327 	pqm_uninit(&process->pqm);
328 err_process_pqm_init:
329 	hash_del_rcu(&process->kfd_processes);
330 	synchronize_rcu();
331 	mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
332 err_mmu_notifier:
333 	mutex_destroy(&process->mutex);
334 	kfd_pasid_free(process->pasid);
335 err_alloc_pasid:
336 	kfree(process->queues);
337 err_alloc_queues:
338 	kfree(process);
339 err_alloc_process:
340 	return ERR_PTR(err);
341 }
342 
343 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
344 							struct kfd_process *p)
345 {
346 	struct kfd_process_device *pdd = NULL;
347 
348 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
349 		if (pdd->dev == dev)
350 			break;
351 
352 	return pdd;
353 }
354 
355 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
356 							struct kfd_process *p)
357 {
358 	struct kfd_process_device *pdd = NULL;
359 
360 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
361 	if (pdd != NULL) {
362 		pdd->dev = dev;
363 		INIT_LIST_HEAD(&pdd->qpd.queues_list);
364 		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
365 		pdd->qpd.dqm = dev->dqm;
366 		pdd->reset_wavefronts = false;
367 		list_add(&pdd->per_device_list, &p->per_device_data);
368 	}
369 
370 	return pdd;
371 }
372 
373 /*
374  * Direct the IOMMU to bind the process (specifically the pasid->mm)
375  * to the device.
376  * Unbinding occurs when the process dies or the device is removed.
377  *
378  * Assumes that the process lock is held.
379  */
380 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
381 							struct kfd_process *p)
382 {
383 	struct kfd_process_device *pdd;
384 	int err;
385 
386 	pdd = kfd_get_process_device_data(dev, p);
387 	if (!pdd) {
388 		pr_err("Process device data doesn't exist\n");
389 		return ERR_PTR(-ENOMEM);
390 	}
391 
392 	if (pdd->bound)
393 		return pdd;
394 
395 	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
396 	if (err < 0)
397 		return ERR_PTR(err);
398 
399 	pdd->bound = true;
400 
401 	return pdd;
402 }
403 
404 void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
405 {
406 	struct kfd_process *p;
407 	struct kfd_process_device *pdd;
408 
409 	BUG_ON(dev == NULL);
410 
411 	/*
412 	 * Look for the process that matches the pasid. If there is no such
413 	 * process, we either released it in amdkfd's own notifier, or there
414 	 * is a bug. Unfortunately, there is no way to tell...
415 	 */
416 	p = kfd_lookup_process_by_pasid(pasid);
417 	if (!p)
418 		return;
419 
420 	pr_debug("Unbinding process %d from IOMMU\n", pasid);
421 
422 	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
423 		kfd_dbgmgr_destroy(dev->dbgmgr);
424 
425 	pqm_uninit(&p->pqm);
426 
427 	pdd = kfd_get_process_device_data(dev, p);
428 
429 	if (!pdd) {
430 		mutex_unlock(&p->mutex);
431 		return;
432 	}
433 
434 	if (pdd->reset_wavefronts) {
435 		dbgdev_wave_reset_wavefronts(pdd->dev, p);
436 		pdd->reset_wavefronts = false;
437 	}
438 
439 	/*
440 	 * Just mark pdd as unbound, because we still need it
441 	 * to call amd_iommu_unbind_pasid() in when the
442 	 * process exits.
443 	 * We don't call amd_iommu_unbind_pasid() here
444 	 * because the IOMMU called us.
445 	 */
446 	pdd->bound = false;
447 
448 	mutex_unlock(&p->mutex);
449 }
450 
451 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
452 {
453 	return list_first_entry(&p->per_device_data,
454 				struct kfd_process_device,
455 				per_device_list);
456 }
457 
458 struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
459 						struct kfd_process_device *pdd)
460 {
461 	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
462 		return NULL;
463 	return list_next_entry(pdd, per_device_list);
464 }
465 
466 bool kfd_has_process_device_data(struct kfd_process *p)
467 {
468 	return !(list_empty(&p->per_device_data));
469 }
470 
471 /* This returns with process->mutex locked. */
472 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
473 {
474 	struct kfd_process *p;
475 	unsigned int temp;
476 
477 	int idx = srcu_read_lock(&kfd_processes_srcu);
478 
479 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
480 		if (p->pasid == pasid) {
481 			mutex_lock(&p->mutex);
482 			break;
483 		}
484 	}
485 
486 	srcu_read_unlock(&kfd_processes_srcu, idx);
487 
488 	return p;
489 }
490