1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/dma-mapping.h>
26 #include <drm/ttm/ttm_range_manager.h>
27 
28 #include "amdgpu.h"
29 #include "amdgpu_vm.h"
30 #include "amdgpu_res_cursor.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "atom.h"
33 
34 struct amdgpu_vram_reservation {
35 	u64 start;
36 	u64 size;
37 	struct list_head allocated;
38 	struct list_head blocks;
39 };
40 
41 static inline struct amdgpu_vram_mgr *
to_vram_mgr(struct ttm_resource_manager * man)42 to_vram_mgr(struct ttm_resource_manager *man)
43 {
44 	return container_of(man, struct amdgpu_vram_mgr, manager);
45 }
46 
47 static inline struct amdgpu_device *
to_amdgpu_device(struct amdgpu_vram_mgr * mgr)48 to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
49 {
50 	return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
51 }
52 
53 static inline struct drm_buddy_block *
amdgpu_vram_mgr_first_block(struct list_head * list)54 amdgpu_vram_mgr_first_block(struct list_head *list)
55 {
56 	return list_first_entry_or_null(list, struct drm_buddy_block, link);
57 }
58 
amdgpu_is_vram_mgr_blocks_contiguous(struct list_head * head)59 static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
60 {
61 	struct drm_buddy_block *block;
62 	u64 start, size;
63 
64 	block = amdgpu_vram_mgr_first_block(head);
65 	if (!block)
66 		return false;
67 
68 	while (head != block->link.next) {
69 		start = amdgpu_vram_mgr_block_start(block);
70 		size = amdgpu_vram_mgr_block_size(block);
71 
72 		block = list_entry(block->link.next, struct drm_buddy_block, link);
73 		if (start + size != amdgpu_vram_mgr_block_start(block))
74 			return false;
75 	}
76 
77 	return true;
78 }
79 
80 
81 
82 /**
83  * DOC: mem_info_vram_total
84  *
85  * The amdgpu driver provides a sysfs API for reporting current total VRAM
86  * available on the device
87  * The file mem_info_vram_total is used for this and returns the total
88  * amount of VRAM in bytes
89  */
amdgpu_mem_info_vram_total_show(struct device * dev,struct device_attribute * attr,char * buf)90 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
91 		struct device_attribute *attr, char *buf)
92 {
93 	struct drm_device *ddev = dev_get_drvdata(dev);
94 	struct amdgpu_device *adev = drm_to_adev(ddev);
95 
96 	return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size);
97 }
98 
99 /**
100  * DOC: mem_info_vis_vram_total
101  *
102  * The amdgpu driver provides a sysfs API for reporting current total
103  * visible VRAM available on the device
104  * The file mem_info_vis_vram_total is used for this and returns the total
105  * amount of visible VRAM in bytes
106  */
amdgpu_mem_info_vis_vram_total_show(struct device * dev,struct device_attribute * attr,char * buf)107 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
108 		struct device_attribute *attr, char *buf)
109 {
110 	struct drm_device *ddev = dev_get_drvdata(dev);
111 	struct amdgpu_device *adev = drm_to_adev(ddev);
112 
113 	return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size);
114 }
115 
116 /**
117  * DOC: mem_info_vram_used
118  *
119  * The amdgpu driver provides a sysfs API for reporting current total VRAM
120  * available on the device
121  * The file mem_info_vram_used is used for this and returns the total
122  * amount of currently used VRAM in bytes
123  */
amdgpu_mem_info_vram_used_show(struct device * dev,struct device_attribute * attr,char * buf)124 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
125 					      struct device_attribute *attr,
126 					      char *buf)
127 {
128 	struct drm_device *ddev = dev_get_drvdata(dev);
129 	struct amdgpu_device *adev = drm_to_adev(ddev);
130 	struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager;
131 
132 	return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man));
133 }
134 
135 /**
136  * DOC: mem_info_vis_vram_used
137  *
138  * The amdgpu driver provides a sysfs API for reporting current total of
139  * used visible VRAM
140  * The file mem_info_vis_vram_used is used for this and returns the total
141  * amount of currently used visible VRAM in bytes
142  */
amdgpu_mem_info_vis_vram_used_show(struct device * dev,struct device_attribute * attr,char * buf)143 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
144 						  struct device_attribute *attr,
145 						  char *buf)
146 {
147 	struct drm_device *ddev = dev_get_drvdata(dev);
148 	struct amdgpu_device *adev = drm_to_adev(ddev);
149 
150 	return sysfs_emit(buf, "%llu\n",
151 			  amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr));
152 }
153 
154 /**
155  * DOC: mem_info_vram_vendor
156  *
157  * The amdgpu driver provides a sysfs API for reporting the vendor of the
158  * installed VRAM
159  * The file mem_info_vram_vendor is used for this and returns the name of the
160  * vendor.
161  */
amdgpu_mem_info_vram_vendor(struct device * dev,struct device_attribute * attr,char * buf)162 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
163 					   struct device_attribute *attr,
164 					   char *buf)
165 {
166 	struct drm_device *ddev = dev_get_drvdata(dev);
167 	struct amdgpu_device *adev = drm_to_adev(ddev);
168 
169 	switch (adev->gmc.vram_vendor) {
170 	case SAMSUNG:
171 		return sysfs_emit(buf, "samsung\n");
172 	case INFINEON:
173 		return sysfs_emit(buf, "infineon\n");
174 	case ELPIDA:
175 		return sysfs_emit(buf, "elpida\n");
176 	case ETRON:
177 		return sysfs_emit(buf, "etron\n");
178 	case NANYA:
179 		return sysfs_emit(buf, "nanya\n");
180 	case HYNIX:
181 		return sysfs_emit(buf, "hynix\n");
182 	case MOSEL:
183 		return sysfs_emit(buf, "mosel\n");
184 	case WINBOND:
185 		return sysfs_emit(buf, "winbond\n");
186 	case ESMT:
187 		return sysfs_emit(buf, "esmt\n");
188 	case MICRON:
189 		return sysfs_emit(buf, "micron\n");
190 	default:
191 		return sysfs_emit(buf, "unknown\n");
192 	}
193 }
194 
195 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
196 		   amdgpu_mem_info_vram_total_show, NULL);
197 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
198 		   amdgpu_mem_info_vis_vram_total_show,NULL);
199 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
200 		   amdgpu_mem_info_vram_used_show, NULL);
201 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
202 		   amdgpu_mem_info_vis_vram_used_show, NULL);
203 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
204 		   amdgpu_mem_info_vram_vendor, NULL);
205 
206 static struct attribute *amdgpu_vram_mgr_attributes[] = {
207 	&dev_attr_mem_info_vram_total.attr,
208 	&dev_attr_mem_info_vis_vram_total.attr,
209 	&dev_attr_mem_info_vram_used.attr,
210 	&dev_attr_mem_info_vis_vram_used.attr,
211 	&dev_attr_mem_info_vram_vendor.attr,
212 	NULL
213 };
214 
215 const struct attribute_group amdgpu_vram_mgr_attr_group = {
216 	.attrs = amdgpu_vram_mgr_attributes
217 };
218 
219 /**
220  * amdgpu_vram_mgr_vis_size - Calculate visible block size
221  *
222  * @adev: amdgpu_device pointer
223  * @block: DRM BUDDY block structure
224  *
225  * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
226  */
amdgpu_vram_mgr_vis_size(struct amdgpu_device * adev,struct drm_buddy_block * block)227 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
228 				    struct drm_buddy_block *block)
229 {
230 	u64 start = amdgpu_vram_mgr_block_start(block);
231 	u64 end = start + amdgpu_vram_mgr_block_size(block);
232 
233 	if (start >= adev->gmc.visible_vram_size)
234 		return 0;
235 
236 	return (end > adev->gmc.visible_vram_size ?
237 		adev->gmc.visible_vram_size : end) - start;
238 }
239 
240 /**
241  * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
242  *
243  * @bo: &amdgpu_bo buffer object (must be in VRAM)
244  *
245  * Returns:
246  * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
247  */
amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo * bo)248 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
249 {
250 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
251 	struct ttm_resource *res = bo->tbo.resource;
252 	struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
253 	struct drm_buddy_block *block;
254 	u64 usage = 0;
255 
256 	if (amdgpu_gmc_vram_full_visible(&adev->gmc))
257 		return amdgpu_bo_size(bo);
258 
259 	if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
260 		return 0;
261 
262 	list_for_each_entry(block, &vres->blocks, link)
263 		usage += amdgpu_vram_mgr_vis_size(adev, block);
264 
265 	return usage;
266 }
267 
268 /* Commit the reservation of VRAM pages */
amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager * man)269 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
270 {
271 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
272 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
273 	struct drm_buddy *mm = &mgr->mm;
274 	struct amdgpu_vram_reservation *rsv, *temp;
275 	struct drm_buddy_block *block;
276 	uint64_t vis_usage;
277 
278 	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
279 		if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
280 					   rsv->size, mm->chunk_size, &rsv->allocated,
281 					   DRM_BUDDY_RANGE_ALLOCATION))
282 			continue;
283 
284 		block = amdgpu_vram_mgr_first_block(&rsv->allocated);
285 		if (!block)
286 			continue;
287 
288 		dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
289 			rsv->start, rsv->size);
290 
291 		vis_usage = amdgpu_vram_mgr_vis_size(adev, block);
292 		atomic64_add(vis_usage, &mgr->vis_usage);
293 		spin_lock(&man->bdev->lru_lock);
294 		man->usage += rsv->size;
295 		spin_unlock(&man->bdev->lru_lock);
296 		list_move(&rsv->blocks, &mgr->reserved_pages);
297 	}
298 }
299 
300 /**
301  * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
302  *
303  * @mgr: amdgpu_vram_mgr pointer
304  * @start: start address of the range in VRAM
305  * @size: size of the range
306  *
307  * Reserve memory from start address with the specified size in VRAM
308  */
amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr * mgr,uint64_t start,uint64_t size)309 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
310 				  uint64_t start, uint64_t size)
311 {
312 	struct amdgpu_vram_reservation *rsv;
313 
314 	rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
315 	if (!rsv)
316 		return -ENOMEM;
317 
318 	INIT_LIST_HEAD(&rsv->allocated);
319 	INIT_LIST_HEAD(&rsv->blocks);
320 
321 	rsv->start = start;
322 	rsv->size = size;
323 
324 	mutex_lock(&mgr->lock);
325 	list_add_tail(&rsv->blocks, &mgr->reservations_pending);
326 	amdgpu_vram_mgr_do_reserve(&mgr->manager);
327 	mutex_unlock(&mgr->lock);
328 
329 	return 0;
330 }
331 
332 /**
333  * amdgpu_vram_mgr_query_page_status - query the reservation status
334  *
335  * @mgr: amdgpu_vram_mgr pointer
336  * @start: start address of a page in VRAM
337  *
338  * Returns:
339  *	-EBUSY: the page is still hold and in pending list
340  *	0: the page has been reserved
341  *	-ENOENT: the input page is not a reservation
342  */
amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr * mgr,uint64_t start)343 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
344 				      uint64_t start)
345 {
346 	struct amdgpu_vram_reservation *rsv;
347 	int ret;
348 
349 	mutex_lock(&mgr->lock);
350 
351 	list_for_each_entry(rsv, &mgr->reservations_pending, blocks) {
352 		if (rsv->start <= start &&
353 		    (start < (rsv->start + rsv->size))) {
354 			ret = -EBUSY;
355 			goto out;
356 		}
357 	}
358 
359 	list_for_each_entry(rsv, &mgr->reserved_pages, blocks) {
360 		if (rsv->start <= start &&
361 		    (start < (rsv->start + rsv->size))) {
362 			ret = 0;
363 			goto out;
364 		}
365 	}
366 
367 	ret = -ENOENT;
368 out:
369 	mutex_unlock(&mgr->lock);
370 	return ret;
371 }
372 
amdgpu_dummy_vram_mgr_debug(struct ttm_resource_manager * man,struct drm_printer * printer)373 static void amdgpu_dummy_vram_mgr_debug(struct ttm_resource_manager *man,
374 				  struct drm_printer *printer)
375 {
376 	DRM_DEBUG_DRIVER("Dummy vram mgr debug\n");
377 }
378 
amdgpu_dummy_vram_mgr_compatible(struct ttm_resource_manager * man,struct ttm_resource * res,const struct ttm_place * place,size_t size)379 static bool amdgpu_dummy_vram_mgr_compatible(struct ttm_resource_manager *man,
380 				       struct ttm_resource *res,
381 				       const struct ttm_place *place,
382 				       size_t size)
383 {
384 	DRM_DEBUG_DRIVER("Dummy vram mgr compatible\n");
385 	return false;
386 }
387 
amdgpu_dummy_vram_mgr_intersects(struct ttm_resource_manager * man,struct ttm_resource * res,const struct ttm_place * place,size_t size)388 static bool amdgpu_dummy_vram_mgr_intersects(struct ttm_resource_manager *man,
389 				       struct ttm_resource *res,
390 				       const struct ttm_place *place,
391 				       size_t size)
392 {
393 	DRM_DEBUG_DRIVER("Dummy vram mgr intersects\n");
394 	return true;
395 }
396 
amdgpu_dummy_vram_mgr_del(struct ttm_resource_manager * man,struct ttm_resource * res)397 static void amdgpu_dummy_vram_mgr_del(struct ttm_resource_manager *man,
398 				struct ttm_resource *res)
399 {
400 	DRM_DEBUG_DRIVER("Dummy vram mgr deleted\n");
401 }
402 
amdgpu_dummy_vram_mgr_new(struct ttm_resource_manager * man,struct ttm_buffer_object * tbo,const struct ttm_place * place,struct ttm_resource ** res)403 static int amdgpu_dummy_vram_mgr_new(struct ttm_resource_manager *man,
404 			       struct ttm_buffer_object *tbo,
405 			       const struct ttm_place *place,
406 			       struct ttm_resource **res)
407 {
408 	DRM_DEBUG_DRIVER("Dummy vram mgr new\n");
409 	return -ENOSPC;
410 }
411 
412 /**
413  * amdgpu_vram_mgr_new - allocate new ranges
414  *
415  * @man: TTM memory type manager
416  * @tbo: TTM BO we need this range for
417  * @place: placement flags and restrictions
418  * @res: the resulting mem object
419  *
420  * Allocate VRAM for the given BO.
421  */
amdgpu_vram_mgr_new(struct ttm_resource_manager * man,struct ttm_buffer_object * tbo,const struct ttm_place * place,struct ttm_resource ** res)422 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
423 			       struct ttm_buffer_object *tbo,
424 			       const struct ttm_place *place,
425 			       struct ttm_resource **res)
426 {
427 	u64 vis_usage = 0, max_bytes, cur_size, min_block_size;
428 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
429 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
430 	struct amdgpu_vram_mgr_resource *vres;
431 	u64 size, remaining_size, lpfn, fpfn;
432 	struct drm_buddy *mm = &mgr->mm;
433 	struct drm_buddy_block *block;
434 	unsigned long pages_per_block;
435 	int r;
436 
437 	lpfn = (u64)place->lpfn << PAGE_SHIFT;
438 	if (!lpfn)
439 		lpfn = man->size;
440 
441 	fpfn = (u64)place->fpfn << PAGE_SHIFT;
442 
443 	max_bytes = adev->gmc.mc_vram_size;
444 	if (tbo->type != ttm_bo_type_kernel)
445 		max_bytes -= AMDGPU_VM_RESERVED_VRAM;
446 
447 	if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
448 		pages_per_block = ~0ul;
449 	} else {
450 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
451 		pages_per_block = HPAGE_PMD_NR;
452 #else
453 		/* default to 2MB */
454 		pages_per_block = 2UL << (20UL - PAGE_SHIFT);
455 #endif
456 		pages_per_block = max_t(uint32_t, pages_per_block,
457 					tbo->page_alignment);
458 	}
459 
460 	vres = kzalloc(sizeof(*vres), GFP_KERNEL);
461 	if (!vres)
462 		return -ENOMEM;
463 
464 	ttm_resource_init(tbo, place, &vres->base);
465 
466 	/* bail out quickly if there's likely not enough VRAM for this BO */
467 	if (ttm_resource_manager_usage(man) > max_bytes) {
468 		r = -ENOSPC;
469 		goto error_fini;
470 	}
471 
472 	INIT_LIST_HEAD(&vres->blocks);
473 
474 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
475 		vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
476 
477 	if (fpfn || lpfn != mgr->mm.size)
478 		/* Allocate blocks in desired range */
479 		vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
480 
481 	remaining_size = (u64)vres->base.size;
482 
483 	mutex_lock(&mgr->lock);
484 	while (remaining_size) {
485 		if (tbo->page_alignment)
486 			min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT;
487 		else
488 			min_block_size = mgr->default_page_size;
489 
490 		BUG_ON(min_block_size < mm->chunk_size);
491 
492 		/* Limit maximum size to 2GiB due to SG table limitations */
493 		size = min(remaining_size, 2ULL << 30);
494 
495 		if ((size >= (u64)pages_per_block << PAGE_SHIFT) &&
496 				!(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))
497 			min_block_size = (u64)pages_per_block << PAGE_SHIFT;
498 
499 		cur_size = size;
500 
501 		if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT) {
502 			/*
503 			 * Except for actual range allocation, modify the size and
504 			 * min_block_size conforming to continuous flag enablement
505 			 */
506 			if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
507 				size = roundup_pow_of_two(size);
508 				min_block_size = size;
509 			/*
510 			 * Modify the size value if size is not
511 			 * aligned with min_block_size
512 			 */
513 			} else if (!IS_ALIGNED(size, min_block_size)) {
514 				size = round_up(size, min_block_size);
515 			}
516 		}
517 
518 		r = drm_buddy_alloc_blocks(mm, fpfn,
519 					   lpfn,
520 					   size,
521 					   min_block_size,
522 					   &vres->blocks,
523 					   vres->flags);
524 		if (unlikely(r))
525 			goto error_free_blocks;
526 
527 		if (size > remaining_size)
528 			remaining_size = 0;
529 		else
530 			remaining_size -= size;
531 	}
532 	mutex_unlock(&mgr->lock);
533 
534 	if (cur_size != size) {
535 		struct drm_buddy_block *block;
536 		struct list_head *trim_list;
537 		u64 original_size;
538 		LIST_HEAD(temp);
539 
540 		trim_list = &vres->blocks;
541 		original_size = (u64)vres->base.size;
542 
543 		/*
544 		 * If size value is rounded up to min_block_size, trim the last
545 		 * block to the required size
546 		 */
547 		if (!list_is_singular(&vres->blocks)) {
548 			block = list_last_entry(&vres->blocks, typeof(*block), link);
549 			list_move_tail(&block->link, &temp);
550 			trim_list = &temp;
551 			/*
552 			 * Compute the original_size value by subtracting the
553 			 * last block size with (aligned size - original size)
554 			 */
555 			original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size);
556 		}
557 
558 		mutex_lock(&mgr->lock);
559 		drm_buddy_block_trim(mm,
560 				     original_size,
561 				     trim_list);
562 		mutex_unlock(&mgr->lock);
563 
564 		if (!list_empty(&temp))
565 			list_splice_tail(trim_list, &vres->blocks);
566 	}
567 
568 	vres->base.start = 0;
569 	list_for_each_entry(block, &vres->blocks, link) {
570 		unsigned long start;
571 
572 		start = amdgpu_vram_mgr_block_start(block) +
573 			amdgpu_vram_mgr_block_size(block);
574 		start >>= PAGE_SHIFT;
575 
576 		if (start > PFN_UP(vres->base.size))
577 			start -= PFN_UP(vres->base.size);
578 		else
579 			start = 0;
580 		vres->base.start = max(vres->base.start, start);
581 
582 		vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
583 	}
584 
585 	if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks))
586 		vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
587 
588 	if (adev->gmc.xgmi.connected_to_cpu)
589 		vres->base.bus.caching = ttm_cached;
590 	else
591 		vres->base.bus.caching = ttm_write_combined;
592 
593 	atomic64_add(vis_usage, &mgr->vis_usage);
594 	*res = &vres->base;
595 	return 0;
596 
597 error_free_blocks:
598 	drm_buddy_free_list(mm, &vres->blocks);
599 	mutex_unlock(&mgr->lock);
600 error_fini:
601 	ttm_resource_fini(man, &vres->base);
602 	kfree(vres);
603 
604 	return r;
605 }
606 
607 /**
608  * amdgpu_vram_mgr_del - free ranges
609  *
610  * @man: TTM memory type manager
611  * @res: TTM memory object
612  *
613  * Free the allocated VRAM again.
614  */
amdgpu_vram_mgr_del(struct ttm_resource_manager * man,struct ttm_resource * res)615 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
616 				struct ttm_resource *res)
617 {
618 	struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
619 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
620 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
621 	struct drm_buddy *mm = &mgr->mm;
622 	struct drm_buddy_block *block;
623 	uint64_t vis_usage = 0;
624 
625 	mutex_lock(&mgr->lock);
626 	list_for_each_entry(block, &vres->blocks, link)
627 		vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
628 
629 	amdgpu_vram_mgr_do_reserve(man);
630 
631 	drm_buddy_free_list(mm, &vres->blocks);
632 	mutex_unlock(&mgr->lock);
633 
634 	atomic64_sub(vis_usage, &mgr->vis_usage);
635 
636 	ttm_resource_fini(man, res);
637 	kfree(vres);
638 }
639 
640 /**
641  * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
642  *
643  * @adev: amdgpu device pointer
644  * @res: TTM memory object
645  * @offset: byte offset from the base of VRAM BO
646  * @length: number of bytes to export in sg_table
647  * @dev: the other device
648  * @dir: dma direction
649  * @sgt: resulting sg table
650  *
651  * Allocate and fill a sg table from a VRAM allocation.
652  */
amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device * adev,struct ttm_resource * res,u64 offset,u64 length,struct device * dev,enum dma_data_direction dir,struct sg_table ** sgt)653 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
654 			      struct ttm_resource *res,
655 			      u64 offset, u64 length,
656 			      struct device *dev,
657 			      enum dma_data_direction dir,
658 			      struct sg_table **sgt)
659 {
660 	struct amdgpu_res_cursor cursor;
661 	struct scatterlist *sg;
662 	int num_entries = 0;
663 	int i, r;
664 
665 	*sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
666 	if (!*sgt)
667 		return -ENOMEM;
668 
669 	/* Determine the number of DRM_BUDDY blocks to export */
670 	amdgpu_res_first(res, offset, length, &cursor);
671 	while (cursor.remaining) {
672 		num_entries++;
673 		amdgpu_res_next(&cursor, cursor.size);
674 	}
675 
676 	r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
677 	if (r)
678 		goto error_free;
679 
680 	/* Initialize scatterlist nodes of sg_table */
681 	for_each_sgtable_sg((*sgt), sg, i)
682 		sg->length = 0;
683 
684 	/*
685 	 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
686 	 * @note: Use iterator api to get first the DRM_BUDDY block
687 	 * and the number of bytes from it. Access the following
688 	 * DRM_BUDDY block(s) if more buffer needs to exported
689 	 */
690 	amdgpu_res_first(res, offset, length, &cursor);
691 	for_each_sgtable_sg((*sgt), sg, i) {
692 		phys_addr_t phys = cursor.start + adev->gmc.aper_base;
693 		size_t size = cursor.size;
694 		dma_addr_t addr;
695 
696 		addr = dma_map_resource(dev, phys, size, dir,
697 					DMA_ATTR_SKIP_CPU_SYNC);
698 		r = dma_mapping_error(dev, addr);
699 		if (r)
700 			goto error_unmap;
701 
702 		sg_set_page(sg, NULL, size, 0);
703 		sg_dma_address(sg) = addr;
704 		sg_dma_len(sg) = size;
705 
706 		amdgpu_res_next(&cursor, cursor.size);
707 	}
708 
709 	return 0;
710 
711 error_unmap:
712 	for_each_sgtable_sg((*sgt), sg, i) {
713 		if (!sg->length)
714 			continue;
715 
716 		dma_unmap_resource(dev, sg->dma_address,
717 				   sg->length, dir,
718 				   DMA_ATTR_SKIP_CPU_SYNC);
719 	}
720 	sg_free_table(*sgt);
721 
722 error_free:
723 	kfree(*sgt);
724 	return r;
725 }
726 
727 /**
728  * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
729  *
730  * @dev: device pointer
731  * @dir: data direction of resource to unmap
732  * @sgt: sg table to free
733  *
734  * Free a previously allocate sg table.
735  */
amdgpu_vram_mgr_free_sgt(struct device * dev,enum dma_data_direction dir,struct sg_table * sgt)736 void amdgpu_vram_mgr_free_sgt(struct device *dev,
737 			      enum dma_data_direction dir,
738 			      struct sg_table *sgt)
739 {
740 	struct scatterlist *sg;
741 	int i;
742 
743 	for_each_sgtable_sg(sgt, sg, i)
744 		dma_unmap_resource(dev, sg->dma_address,
745 				   sg->length, dir,
746 				   DMA_ATTR_SKIP_CPU_SYNC);
747 	sg_free_table(sgt);
748 	kfree(sgt);
749 }
750 
751 /**
752  * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
753  *
754  * @mgr: amdgpu_vram_mgr pointer
755  *
756  * Returns how many bytes are used in the visible part of VRAM
757  */
amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr * mgr)758 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
759 {
760 	return atomic64_read(&mgr->vis_usage);
761 }
762 
763 /**
764  * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection
765  *
766  * @man: TTM memory type manager
767  * @res: The resource to test
768  * @place: The place to test against
769  * @size: Size of the new allocation
770  *
771  * Test each drm buddy block for intersection for eviction decision.
772  */
amdgpu_vram_mgr_intersects(struct ttm_resource_manager * man,struct ttm_resource * res,const struct ttm_place * place,size_t size)773 static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
774 				       struct ttm_resource *res,
775 				       const struct ttm_place *place,
776 				       size_t size)
777 {
778 	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
779 	struct drm_buddy_block *block;
780 
781 	/* Check each drm buddy block individually */
782 	list_for_each_entry(block, &mgr->blocks, link) {
783 		unsigned long fpfn =
784 			amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
785 		unsigned long lpfn = fpfn +
786 			(amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
787 
788 		if (place->fpfn < lpfn &&
789 		    (!place->lpfn || place->lpfn > fpfn))
790 			return true;
791 	}
792 
793 	return false;
794 }
795 
796 /**
797  * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility
798  *
799  * @man: TTM memory type manager
800  * @res: The resource to test
801  * @place: The place to test against
802  * @size: Size of the new allocation
803  *
804  * Test each drm buddy block for placement compatibility.
805  */
amdgpu_vram_mgr_compatible(struct ttm_resource_manager * man,struct ttm_resource * res,const struct ttm_place * place,size_t size)806 static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
807 				       struct ttm_resource *res,
808 				       const struct ttm_place *place,
809 				       size_t size)
810 {
811 	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
812 	struct drm_buddy_block *block;
813 
814 	/* Check each drm buddy block individually */
815 	list_for_each_entry(block, &mgr->blocks, link) {
816 		unsigned long fpfn =
817 			amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
818 		unsigned long lpfn = fpfn +
819 			(amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
820 
821 		if (fpfn < place->fpfn ||
822 		    (place->lpfn && lpfn > place->lpfn))
823 			return false;
824 	}
825 
826 	return true;
827 }
828 
829 /**
830  * amdgpu_vram_mgr_debug - dump VRAM table
831  *
832  * @man: TTM memory type manager
833  * @printer: DRM printer to use
834  *
835  * Dump the table content using printk.
836  */
amdgpu_vram_mgr_debug(struct ttm_resource_manager * man,struct drm_printer * printer)837 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
838 				  struct drm_printer *printer)
839 {
840 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
841 	struct drm_buddy *mm = &mgr->mm;
842 	struct amdgpu_vram_reservation *rsv;
843 
844 	drm_printf(printer, "  vis usage:%llu\n",
845 		   amdgpu_vram_mgr_vis_usage(mgr));
846 
847 	mutex_lock(&mgr->lock);
848 	drm_printf(printer, "default_page_size: %lluKiB\n",
849 		   mgr->default_page_size >> 10);
850 
851 	drm_buddy_print(mm, printer);
852 
853 	drm_printf(printer, "reserved:\n");
854 	list_for_each_entry(rsv, &mgr->reserved_pages, blocks)
855 		drm_printf(printer, "%#018llx-%#018llx: %llu\n",
856 			rsv->start, rsv->start + rsv->size, rsv->size);
857 	mutex_unlock(&mgr->lock);
858 }
859 
860 static const struct ttm_resource_manager_func amdgpu_dummy_vram_mgr_func = {
861 	.alloc	= amdgpu_dummy_vram_mgr_new,
862 	.free	= amdgpu_dummy_vram_mgr_del,
863 	.intersects = amdgpu_dummy_vram_mgr_intersects,
864 	.compatible = amdgpu_dummy_vram_mgr_compatible,
865 	.debug	= amdgpu_dummy_vram_mgr_debug
866 };
867 
868 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
869 	.alloc	= amdgpu_vram_mgr_new,
870 	.free	= amdgpu_vram_mgr_del,
871 	.intersects = amdgpu_vram_mgr_intersects,
872 	.compatible = amdgpu_vram_mgr_compatible,
873 	.debug	= amdgpu_vram_mgr_debug
874 };
875 
876 /**
877  * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
878  *
879  * @adev: amdgpu_device pointer
880  *
881  * Allocate and initialize the VRAM manager.
882  */
amdgpu_vram_mgr_init(struct amdgpu_device * adev)883 int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
884 {
885 	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
886 	struct ttm_resource_manager *man = &mgr->manager;
887 	int err;
888 
889 	ttm_resource_manager_init(man, &adev->mman.bdev,
890 				  adev->gmc.real_vram_size);
891 
892 	mutex_init(&mgr->lock);
893 	INIT_LIST_HEAD(&mgr->reservations_pending);
894 	INIT_LIST_HEAD(&mgr->reserved_pages);
895 	mgr->default_page_size = PAGE_SIZE;
896 
897 	if (!adev->gmc.is_app_apu) {
898 		man->func = &amdgpu_vram_mgr_func;
899 
900 		err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
901 		if (err)
902 			return err;
903 	} else {
904 		man->func = &amdgpu_dummy_vram_mgr_func;
905 		DRM_INFO("Setup dummy vram mgr\n");
906 	}
907 
908 	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
909 	ttm_resource_manager_set_used(man, true);
910 	return 0;
911 }
912 
913 /**
914  * amdgpu_vram_mgr_fini - free and destroy VRAM manager
915  *
916  * @adev: amdgpu_device pointer
917  *
918  * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
919  * allocated inside it.
920  */
amdgpu_vram_mgr_fini(struct amdgpu_device * adev)921 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
922 {
923 	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
924 	struct ttm_resource_manager *man = &mgr->manager;
925 	int ret;
926 	struct amdgpu_vram_reservation *rsv, *temp;
927 
928 	ttm_resource_manager_set_used(man, false);
929 
930 	ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
931 	if (ret)
932 		return;
933 
934 	mutex_lock(&mgr->lock);
935 	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)
936 		kfree(rsv);
937 
938 	list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
939 		drm_buddy_free_list(&mgr->mm, &rsv->allocated);
940 		kfree(rsv);
941 	}
942 	if (!adev->gmc.is_app_apu)
943 		drm_buddy_fini(&mgr->mm);
944 	mutex_unlock(&mgr->lock);
945 
946 	ttm_resource_manager_cleanup(man);
947 	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
948 }
949