1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/amd-iommu.h>
24 #include <linux/bsearch.h>
25 #include <linux/pci.h>
26 #include <linux/slab.h>
27 #include "kfd_priv.h"
28 #include "kfd_device_queue_manager.h"
29 #include "kfd_pm4_headers_vi.h"
30 
31 #define MQD_SIZE_ALIGNED 768
32 
33 static const struct kfd_device_info kaveri_device_info = {
34 	.asic_family = CHIP_KAVERI,
35 	.max_pasid_bits = 16,
36 	/* max num of queues for KV.TODO should be a dynamic value */
37 	.max_no_of_hqd	= 24,
38 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
39 	.event_interrupt_class = &event_interrupt_class_cik,
40 	.num_of_watch_points = 4,
41 	.mqd_size_aligned = MQD_SIZE_ALIGNED
42 };
43 
44 static const struct kfd_device_info carrizo_device_info = {
45 	.asic_family = CHIP_CARRIZO,
46 	.max_pasid_bits = 16,
47 	/* max num of queues for CZ.TODO should be a dynamic value */
48 	.max_no_of_hqd	= 24,
49 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
50 	.event_interrupt_class = &event_interrupt_class_cik,
51 	.num_of_watch_points = 4,
52 	.mqd_size_aligned = MQD_SIZE_ALIGNED
53 };
54 
55 struct kfd_deviceid {
56 	unsigned short did;
57 	const struct kfd_device_info *device_info;
58 };
59 
60 /* Please keep this sorted by increasing device id. */
61 static const struct kfd_deviceid supported_devices[] = {
62 	{ 0x1304, &kaveri_device_info },	/* Kaveri */
63 	{ 0x1305, &kaveri_device_info },	/* Kaveri */
64 	{ 0x1306, &kaveri_device_info },	/* Kaveri */
65 	{ 0x1307, &kaveri_device_info },	/* Kaveri */
66 	{ 0x1309, &kaveri_device_info },	/* Kaveri */
67 	{ 0x130A, &kaveri_device_info },	/* Kaveri */
68 	{ 0x130B, &kaveri_device_info },	/* Kaveri */
69 	{ 0x130C, &kaveri_device_info },	/* Kaveri */
70 	{ 0x130D, &kaveri_device_info },	/* Kaveri */
71 	{ 0x130E, &kaveri_device_info },	/* Kaveri */
72 	{ 0x130F, &kaveri_device_info },	/* Kaveri */
73 	{ 0x1310, &kaveri_device_info },	/* Kaveri */
74 	{ 0x1311, &kaveri_device_info },	/* Kaveri */
75 	{ 0x1312, &kaveri_device_info },	/* Kaveri */
76 	{ 0x1313, &kaveri_device_info },	/* Kaveri */
77 	{ 0x1315, &kaveri_device_info },	/* Kaveri */
78 	{ 0x1316, &kaveri_device_info },	/* Kaveri */
79 	{ 0x1317, &kaveri_device_info },	/* Kaveri */
80 	{ 0x1318, &kaveri_device_info },	/* Kaveri */
81 	{ 0x131B, &kaveri_device_info },	/* Kaveri */
82 	{ 0x131C, &kaveri_device_info },	/* Kaveri */
83 	{ 0x131D, &kaveri_device_info },	/* Kaveri */
84 	{ 0x9870, &carrizo_device_info },	/* Carrizo */
85 	{ 0x9874, &carrizo_device_info },	/* Carrizo */
86 	{ 0x9875, &carrizo_device_info },	/* Carrizo */
87 	{ 0x9876, &carrizo_device_info },	/* Carrizo */
88 	{ 0x9877, &carrizo_device_info }	/* Carrizo */
89 };
90 
91 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
92 				unsigned int chunk_size);
93 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
94 
95 static int kfd_resume(struct kfd_dev *kfd);
96 
97 static const struct kfd_device_info *lookup_device_info(unsigned short did)
98 {
99 	size_t i;
100 
101 	for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
102 		if (supported_devices[i].did == did) {
103 			WARN_ON(!supported_devices[i].device_info);
104 			return supported_devices[i].device_info;
105 		}
106 	}
107 
108 	dev_warn(kfd_device, "DID %04x is missing in supported_devices\n",
109 		 did);
110 
111 	return NULL;
112 }
113 
114 struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
115 	struct pci_dev *pdev, const struct kfd2kgd_calls *f2g)
116 {
117 	struct kfd_dev *kfd;
118 
119 	const struct kfd_device_info *device_info =
120 					lookup_device_info(pdev->device);
121 
122 	if (!device_info) {
123 		dev_err(kfd_device, "kgd2kfd_probe failed\n");
124 		return NULL;
125 	}
126 
127 	kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
128 	if (!kfd)
129 		return NULL;
130 
131 	kfd->kgd = kgd;
132 	kfd->device_info = device_info;
133 	kfd->pdev = pdev;
134 	kfd->init_complete = false;
135 	kfd->kfd2kgd = f2g;
136 
137 	mutex_init(&kfd->doorbell_mutex);
138 	memset(&kfd->doorbell_available_index, 0,
139 		sizeof(kfd->doorbell_available_index));
140 
141 	return kfd;
142 }
143 
144 static bool device_iommu_pasid_init(struct kfd_dev *kfd)
145 {
146 	const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
147 					AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
148 					AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
149 
150 	struct amd_iommu_device_info iommu_info;
151 	unsigned int pasid_limit;
152 	int err;
153 
154 	err = amd_iommu_device_info(kfd->pdev, &iommu_info);
155 	if (err < 0) {
156 		dev_err(kfd_device,
157 			"error getting iommu info. is the iommu enabled?\n");
158 		return false;
159 	}
160 
161 	if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
162 		dev_err(kfd_device, "error required iommu flags ats %i, pri %i, pasid %i\n",
163 		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
164 		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
165 		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
166 									!= 0);
167 		return false;
168 	}
169 
170 	pasid_limit = min_t(unsigned int,
171 			(unsigned int)(1 << kfd->device_info->max_pasid_bits),
172 			iommu_info.max_pasids);
173 
174 	if (!kfd_set_pasid_limit(pasid_limit)) {
175 		dev_err(kfd_device, "error setting pasid limit\n");
176 		return false;
177 	}
178 
179 	return true;
180 }
181 
182 static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
183 {
184 	struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
185 
186 	if (dev)
187 		kfd_process_iommu_unbind_callback(dev, pasid);
188 }
189 
190 /*
191  * This function called by IOMMU driver on PPR failure
192  */
193 static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
194 		unsigned long address, u16 flags)
195 {
196 	struct kfd_dev *dev;
197 
198 	dev_warn(kfd_device,
199 			"Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",
200 			PCI_BUS_NUM(pdev->devfn),
201 			PCI_SLOT(pdev->devfn),
202 			PCI_FUNC(pdev->devfn),
203 			pasid,
204 			address,
205 			flags);
206 
207 	dev = kfd_device_by_pci_dev(pdev);
208 	if (!WARN_ON(!dev))
209 		kfd_signal_iommu_event(dev, pasid, address,
210 			flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
211 
212 	return AMD_IOMMU_INV_PRI_RSP_INVALID;
213 }
214 
215 bool kgd2kfd_device_init(struct kfd_dev *kfd,
216 			 const struct kgd2kfd_shared_resources *gpu_resources)
217 {
218 	unsigned int size;
219 
220 	kfd->shared_resources = *gpu_resources;
221 
222 	kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
223 	kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
224 	kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
225 			- kfd->vm_info.first_vmid_kfd + 1;
226 
227 	/* calculate max size of mqds needed for queues */
228 	size = max_num_of_queues_per_device *
229 			kfd->device_info->mqd_size_aligned;
230 
231 	/*
232 	 * calculate max size of runlist packet.
233 	 * There can be only 2 packets at once
234 	 */
235 	size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
236 		max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
237 		+ sizeof(struct pm4_mes_runlist)) * 2;
238 
239 	/* Add size of HIQ & DIQ */
240 	size += KFD_KERNEL_QUEUE_SIZE * 2;
241 
242 	/* add another 512KB for all other allocations on gart (HPD, fences) */
243 	size += 512 * 1024;
244 
245 	if (kfd->kfd2kgd->init_gtt_mem_allocation(
246 			kfd->kgd, size, &kfd->gtt_mem,
247 			&kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr)){
248 		dev_err(kfd_device, "Could not allocate %d bytes\n", size);
249 		goto out;
250 	}
251 
252 	dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
253 
254 	/* Initialize GTT sa with 512 byte chunk size */
255 	if (kfd_gtt_sa_init(kfd, size, 512) != 0) {
256 		dev_err(kfd_device, "Error initializing gtt sub-allocator\n");
257 		goto kfd_gtt_sa_init_error;
258 	}
259 
260 	if (kfd_doorbell_init(kfd)) {
261 		dev_err(kfd_device,
262 			"Error initializing doorbell aperture\n");
263 		goto kfd_doorbell_error;
264 	}
265 
266 	if (kfd_topology_add_device(kfd)) {
267 		dev_err(kfd_device, "Error adding device to topology\n");
268 		goto kfd_topology_add_device_error;
269 	}
270 
271 	if (kfd_interrupt_init(kfd)) {
272 		dev_err(kfd_device, "Error initializing interrupts\n");
273 		goto kfd_interrupt_error;
274 	}
275 
276 	kfd->dqm = device_queue_manager_init(kfd);
277 	if (!kfd->dqm) {
278 		dev_err(kfd_device, "Error initializing queue manager\n");
279 		goto device_queue_manager_error;
280 	}
281 
282 	if (!device_iommu_pasid_init(kfd)) {
283 		dev_err(kfd_device,
284 			"Error initializing iommuv2 for device %x:%x\n",
285 			kfd->pdev->vendor, kfd->pdev->device);
286 		goto device_iommu_pasid_error;
287 	}
288 
289 	if (kfd_resume(kfd))
290 		goto kfd_resume_error;
291 
292 	kfd->dbgmgr = NULL;
293 
294 	kfd->init_complete = true;
295 	dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
296 		 kfd->pdev->device);
297 
298 	pr_debug("Starting kfd with the following scheduling policy %d\n",
299 		sched_policy);
300 
301 	goto out;
302 
303 kfd_resume_error:
304 device_iommu_pasid_error:
305 	device_queue_manager_uninit(kfd->dqm);
306 device_queue_manager_error:
307 	kfd_interrupt_exit(kfd);
308 kfd_interrupt_error:
309 	kfd_topology_remove_device(kfd);
310 kfd_topology_add_device_error:
311 	kfd_doorbell_fini(kfd);
312 kfd_doorbell_error:
313 	kfd_gtt_sa_fini(kfd);
314 kfd_gtt_sa_init_error:
315 	kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
316 	dev_err(kfd_device,
317 		"device %x:%x NOT added due to errors\n",
318 		kfd->pdev->vendor, kfd->pdev->device);
319 out:
320 	return kfd->init_complete;
321 }
322 
323 void kgd2kfd_device_exit(struct kfd_dev *kfd)
324 {
325 	if (kfd->init_complete) {
326 		kgd2kfd_suspend(kfd);
327 		device_queue_manager_uninit(kfd->dqm);
328 		kfd_interrupt_exit(kfd);
329 		kfd_topology_remove_device(kfd);
330 		kfd_doorbell_fini(kfd);
331 		kfd_gtt_sa_fini(kfd);
332 		kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
333 	}
334 
335 	kfree(kfd);
336 }
337 
338 void kgd2kfd_suspend(struct kfd_dev *kfd)
339 {
340 	if (!kfd->init_complete)
341 		return;
342 
343 	kfd->dqm->ops.stop(kfd->dqm);
344 
345 	kfd_unbind_processes_from_device(kfd);
346 
347 	amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
348 	amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
349 	amd_iommu_free_device(kfd->pdev);
350 }
351 
352 int kgd2kfd_resume(struct kfd_dev *kfd)
353 {
354 	if (!kfd->init_complete)
355 		return 0;
356 
357 	return kfd_resume(kfd);
358 
359 }
360 
361 static int kfd_resume(struct kfd_dev *kfd)
362 {
363 	int err = 0;
364 	unsigned int pasid_limit = kfd_get_pasid_limit();
365 
366 	err = amd_iommu_init_device(kfd->pdev, pasid_limit);
367 	if (err)
368 		return -ENXIO;
369 	amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
370 					iommu_pasid_shutdown_callback);
371 	amd_iommu_set_invalid_ppr_cb(kfd->pdev,
372 				     iommu_invalid_ppr_cb);
373 
374 	err = kfd_bind_processes_to_device(kfd);
375 	if (err)
376 		goto processes_bind_error;
377 
378 	err = kfd->dqm->ops.start(kfd->dqm);
379 	if (err) {
380 		dev_err(kfd_device,
381 			"Error starting queue manager for device %x:%x\n",
382 			kfd->pdev->vendor, kfd->pdev->device);
383 		goto dqm_start_error;
384 	}
385 
386 	return err;
387 
388 dqm_start_error:
389 processes_bind_error:
390 	amd_iommu_free_device(kfd->pdev);
391 
392 	return err;
393 }
394 
395 /* This is called directly from KGD at ISR. */
396 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
397 {
398 	if (!kfd->init_complete)
399 		return;
400 
401 	spin_lock(&kfd->interrupt_lock);
402 
403 	if (kfd->interrupts_active
404 	    && interrupt_is_wanted(kfd, ih_ring_entry)
405 	    && enqueue_ih_ring_entry(kfd, ih_ring_entry))
406 		queue_work(kfd->ih_wq, &kfd->interrupt_work);
407 
408 	spin_unlock(&kfd->interrupt_lock);
409 }
410 
411 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
412 				unsigned int chunk_size)
413 {
414 	unsigned int num_of_longs;
415 
416 	if (WARN_ON(buf_size < chunk_size))
417 		return -EINVAL;
418 	if (WARN_ON(buf_size == 0))
419 		return -EINVAL;
420 	if (WARN_ON(chunk_size == 0))
421 		return -EINVAL;
422 
423 	kfd->gtt_sa_chunk_size = chunk_size;
424 	kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;
425 
426 	num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) /
427 		BITS_PER_LONG;
428 
429 	kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL);
430 
431 	if (!kfd->gtt_sa_bitmap)
432 		return -ENOMEM;
433 
434 	pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
435 			kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);
436 
437 	mutex_init(&kfd->gtt_sa_lock);
438 
439 	return 0;
440 
441 }
442 
443 static void kfd_gtt_sa_fini(struct kfd_dev *kfd)
444 {
445 	mutex_destroy(&kfd->gtt_sa_lock);
446 	kfree(kfd->gtt_sa_bitmap);
447 }
448 
449 static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr,
450 						unsigned int bit_num,
451 						unsigned int chunk_size)
452 {
453 	return start_addr + bit_num * chunk_size;
454 }
455 
456 static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr,
457 						unsigned int bit_num,
458 						unsigned int chunk_size)
459 {
460 	return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size);
461 }
462 
463 int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
464 			struct kfd_mem_obj **mem_obj)
465 {
466 	unsigned int found, start_search, cur_size;
467 
468 	if (size == 0)
469 		return -EINVAL;
470 
471 	if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
472 		return -ENOMEM;
473 
474 	*mem_obj = kmalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
475 	if ((*mem_obj) == NULL)
476 		return -ENOMEM;
477 
478 	pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
479 
480 	start_search = 0;
481 
482 	mutex_lock(&kfd->gtt_sa_lock);
483 
484 kfd_gtt_restart_search:
485 	/* Find the first chunk that is free */
486 	found = find_next_zero_bit(kfd->gtt_sa_bitmap,
487 					kfd->gtt_sa_num_of_chunks,
488 					start_search);
489 
490 	pr_debug("Found = %d\n", found);
491 
492 	/* If there wasn't any free chunk, bail out */
493 	if (found == kfd->gtt_sa_num_of_chunks)
494 		goto kfd_gtt_no_free_chunk;
495 
496 	/* Update fields of mem_obj */
497 	(*mem_obj)->range_start = found;
498 	(*mem_obj)->range_end = found;
499 	(*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr(
500 					kfd->gtt_start_gpu_addr,
501 					found,
502 					kfd->gtt_sa_chunk_size);
503 	(*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr(
504 					kfd->gtt_start_cpu_ptr,
505 					found,
506 					kfd->gtt_sa_chunk_size);
507 
508 	pr_debug("gpu_addr = %p, cpu_addr = %p\n",
509 			(uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr);
510 
511 	/* If we need only one chunk, mark it as allocated and get out */
512 	if (size <= kfd->gtt_sa_chunk_size) {
513 		pr_debug("Single bit\n");
514 		set_bit(found, kfd->gtt_sa_bitmap);
515 		goto kfd_gtt_out;
516 	}
517 
518 	/* Otherwise, try to see if we have enough contiguous chunks */
519 	cur_size = size - kfd->gtt_sa_chunk_size;
520 	do {
521 		(*mem_obj)->range_end =
522 			find_next_zero_bit(kfd->gtt_sa_bitmap,
523 					kfd->gtt_sa_num_of_chunks, ++found);
524 		/*
525 		 * If next free chunk is not contiguous than we need to
526 		 * restart our search from the last free chunk we found (which
527 		 * wasn't contiguous to the previous ones
528 		 */
529 		if ((*mem_obj)->range_end != found) {
530 			start_search = found;
531 			goto kfd_gtt_restart_search;
532 		}
533 
534 		/*
535 		 * If we reached end of buffer, bail out with error
536 		 */
537 		if (found == kfd->gtt_sa_num_of_chunks)
538 			goto kfd_gtt_no_free_chunk;
539 
540 		/* Check if we don't need another chunk */
541 		if (cur_size <= kfd->gtt_sa_chunk_size)
542 			cur_size = 0;
543 		else
544 			cur_size -= kfd->gtt_sa_chunk_size;
545 
546 	} while (cur_size > 0);
547 
548 	pr_debug("range_start = %d, range_end = %d\n",
549 		(*mem_obj)->range_start, (*mem_obj)->range_end);
550 
551 	/* Mark the chunks as allocated */
552 	for (found = (*mem_obj)->range_start;
553 		found <= (*mem_obj)->range_end;
554 		found++)
555 		set_bit(found, kfd->gtt_sa_bitmap);
556 
557 kfd_gtt_out:
558 	mutex_unlock(&kfd->gtt_sa_lock);
559 	return 0;
560 
561 kfd_gtt_no_free_chunk:
562 	pr_debug("Allocation failed with mem_obj = %p\n", mem_obj);
563 	mutex_unlock(&kfd->gtt_sa_lock);
564 	kfree(mem_obj);
565 	return -ENOMEM;
566 }
567 
568 int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
569 {
570 	unsigned int bit;
571 
572 	/* Act like kfree when trying to free a NULL object */
573 	if (!mem_obj)
574 		return 0;
575 
576 	pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n",
577 			mem_obj, mem_obj->range_start, mem_obj->range_end);
578 
579 	mutex_lock(&kfd->gtt_sa_lock);
580 
581 	/* Mark the chunks as free */
582 	for (bit = mem_obj->range_start;
583 		bit <= mem_obj->range_end;
584 		bit++)
585 		clear_bit(bit, kfd->gtt_sa_bitmap);
586 
587 	mutex_unlock(&kfd->gtt_sa_lock);
588 
589 	kfree(mem_obj);
590 	return 0;
591 }
592